diff --git a/CMakeLists.txt b/CMakeLists.txt index b0680a782cf7f..9002cb287e855 100755 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -17,9 +17,12 @@ if(APPLE AND WITH_ARM) cmake_minimum_required(VERSION 3.19.2) cmake_policy(VERSION 3.19.2) else(APPLE AND WITH_ARM) - cmake_minimum_required(VERSION 3.10) + cmake_minimum_required(VERSION 3.15) cmake_policy(VERSION 3.10) endif(APPLE AND WITH_ARM) +# use to get_property location of static lib +# https://cmake.org/cmake/help/v3.0/policy/CMP0026.html?highlight=cmp0026 +cmake_policy(SET CMP0026 OLD) set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_CURRENT_SOURCE_DIR}/cmake") set(PADDLE_SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}) set(PADDLE_BINARY_DIR ${CMAKE_CURRENT_BINARY_DIR}) diff --git a/cmake/external/boost.cmake b/cmake/external/boost.cmake index 69eb62bfdc654..e47b608341bee 100644 --- a/cmake/external/boost.cmake +++ b/cmake/external/boost.cmake @@ -32,7 +32,6 @@ set(BOOST_URL "http://paddlepaddledeps.bj.bcebos.com/${BOOST_TAR}.tar.gz" CACH MESSAGE(STATUS "BOOST_VERSION: ${BOOST_VER}, BOOST_URL: ${BOOST_URL}") set(BOOST_PREFIX_DIR ${THIRD_PARTY_PATH}/boost) - set(BOOST_INCLUDE_DIR "${THIRD_PARTY_PATH}/boost/src/extern_boost" CACHE PATH "boost include directory." FORCE) set_directory_properties(PROPERTIES CLEAN_NO_CUSTOM 1) include_directories(${BOOST_INCLUDE_DIR}) diff --git a/cmake/external/paddle2onnx.cmake b/cmake/external/paddle2onnx.cmake index ba6f0396008fc..2fc22578cae9d 100644 --- a/cmake/external/paddle2onnx.cmake +++ b/cmake/external/paddle2onnx.cmake @@ -53,6 +53,7 @@ set(PADDLE2ONNX_OPTIONAL_ARGS -DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER} -DCMAKE_C_COMPILER=${CMAKE_C_COMPILER} -DCMAKE_CXX_FLAGS=${CMAKE_CXX_FLAGS} + -DCMAKE_CXX_STANDARD=14 -DCMAKE_CXX_FLAGS_RELEASE=${CMAKE_CXX_FLAGS_RELEASE} -DCMAKE_CXX_FLAGS_DEBUG=${CMAKE_CXX_FLAGS_DEBUG} -DCMAKE_C_FLAGS=${CMAKE_C_FLAGS} @@ -60,6 +61,7 @@ set(PADDLE2ONNX_OPTIONAL_ARGS -DCMAKE_C_FLAGS_RELEASE=${CMAKE_C_FLAGS_RELEASE} -DONNX_CUSTOM_PROTOC_PATH=${PROTOC_BIN_PATH} -DWITH_STATIC=OFF + -DMSVC_STATIC_CRT=${MSVC_STATIC_CRT} -DCMAKE_INSTALL_PREFIX=${PADDLE2ONNX_INSTALL_DIR} -DCMAKE_INSTALL_LIBDIR=${PADDLE2ONNX_INSTALL_DIR}/${LIBDIR} -DCMAKE_POSITION_INDEPENDENT_CODE=ON diff --git a/cmake/external/xpu.cmake b/cmake/external/xpu.cmake index cda8029bfe4e4..be911eb7eaced 100644 --- a/cmake/external/xpu.cmake +++ b/cmake/external/xpu.cmake @@ -17,7 +17,7 @@ endif() # ubuntu and centos: use output by XDNN API team if(NOT DEFINED XPU_XDNN_BASE_URL) SET(XPU_XDNN_BASE_URL_WITHOUT_DATE "https://klx-sdk-release-public.su.bcebos.com/xdnn/dev") - SET(XPU_XDNN_BASE_URL "${XPU_XDNN_BASE_URL_WITHOUT_DATE}/20220412") + SET(XPU_XDNN_BASE_URL "${XPU_XDNN_BASE_URL_WITHOUT_DATE}/20220425") else() SET(XPU_XDNN_BASE_URL "${XPU_XDNN_BASE_URL}") endif() diff --git a/cmake/flags.cmake b/cmake/flags.cmake index 5742a6b602ff3..c1a7ba6d909e1 100644 --- a/cmake/flags.cmake +++ b/cmake/flags.cmake @@ -163,7 +163,6 @@ if(NOT APPLE) set(COMMON_FLAGS ${COMMON_FLAGS} -Wno-format-truncation # Warning in boost gcc 8.2 - -Wno-error=cast-function-type # Warning in boost gcc 8.2 -Wno-error=parentheses # Warning in boost gcc 8.2 -Wno-error=catch-value # Warning in boost gcc 8.2 -Wno-error=nonnull-compare # Warning in boost gcc 8.2 diff --git a/cmake/generic.cmake b/cmake/generic.cmake index ba59eae392c66..35170b5198dc3 100644 --- a/cmake/generic.cmake +++ b/cmake/generic.cmake @@ -176,6 +176,36 @@ function(create_static_lib TARGET_NAME) endif() endfunction() +function(create_dummy_static_lib TARGET_NAME) + set(options "") + set(oneValueArgs "") + set(multiValueArgs LIBS DEPS LIMIT) + cmake_parse_arguments(merge "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN}) + + list(REMOVE_DUPLICATES merge_LIBS) + set(index 1) + set(offset 1) + # the dummy target would be consisted of limit size libraries + set(limit ${merge_LIMIT}) + list(LENGTH merge_LIBS libs_len) + foreach(lib ${merge_LIBS}) + list(APPEND merge_list ${lib}) + list(LENGTH merge_list listlen) + if ((${listlen} GREATER ${limit}) OR (${offset} EQUAL ${libs_len})) + message("Merge and generate static library: ${TARGET_NAME}_static_${index}") + merge_static_libs(${TARGET_NAME}_static_${index} ${merge_list}) + if(merge_DEPS) + target_link_libraries(${TARGET_NAME}_static_${index} ${merge_DEPS}) + endif() + set(merge_list) + list(APPEND ${TARGET_NAME}_list ${TARGET_NAME}_static_${index}) + MATH(EXPR index "${index}+1") + endif() + MATH(EXPR offset "${offset}+1") + endforeach() + cc_library(${TARGET_NAME} DEPS ${${TARGET_NAME}_list}) +endfunction() + function(merge_static_libs TARGET_NAME) set(libs ${ARGN}) list(REMOVE_DUPLICATES libs) @@ -193,92 +223,61 @@ function(merge_static_libs TARGET_NAME) # also help to track dependencies. set(target_SRCS ${CMAKE_CURRENT_BINARY_DIR}/${TARGET_NAME}_dummy.c) - if(APPLE) # Use OSX's libtool to merge archives - # Make the generated dummy source file depended on all static input - # libs. If input lib changes,the source file is touched - # which causes the desired effect (relink). - add_custom_command(OUTPUT ${target_SRCS} - COMMAND ${CMAKE_COMMAND} -E touch ${target_SRCS} - DEPENDS ${libs}) - - # Generate dummy static lib - generate_dummy_static_lib(LIB_NAME ${TARGET_NAME} FILE_PATH ${target_SRCS} GENERATOR "generic.cmake:merge_static_libs") - - target_link_libraries(${TARGET_NAME} ${libs_deps}) + # Make the generated dummy source file depended on all static input + # libs. If input lib changes,the source file is touched + # which causes the desired effect (relink). + add_custom_command(OUTPUT ${target_SRCS} + COMMAND ${CMAKE_COMMAND} -E touch ${target_SRCS} + DEPENDS ${libs}) + + # Generate dummy staic lib + generate_dummy_static_lib(LIB_NAME ${TARGET_NAME} FILE_PATH ${target_SRCS} GENERATOR "generic.cmake:merge_static_libs") + target_link_libraries(${TARGET_NAME} ${libs_deps}) + # OSX: use 'libtool' to merge archives + if(APPLE) foreach(lib ${libs}) # Get the file names of the libraries to be merged set(libfiles ${libfiles} $) endforeach() add_custom_command(TARGET ${TARGET_NAME} POST_BUILD + COMMENT "Merge and generate static lib: lib${TARGET_NAME}.a" COMMAND rm "${CMAKE_CURRENT_BINARY_DIR}/lib${TARGET_NAME}.a" COMMAND /usr/bin/libtool -static -o "${CMAKE_CURRENT_BINARY_DIR}/lib${TARGET_NAME}.a" ${libfiles} ) - endif(APPLE) - if(LINUX) # general UNIX: use "ar" to extract objects and re-add to a common lib - set(target_DIR ${CMAKE_CURRENT_BINARY_DIR}/${TARGET_NAME}.dir) - - foreach(lib ${libs}) - set(objlistfile ${target_DIR}/${lib}.objlist) # list of objects in the input library - set(objdir ${target_DIR}/${lib}.objdir) - - add_custom_command(OUTPUT ${objdir} - COMMAND ${CMAKE_COMMAND} -E make_directory ${objdir} - DEPENDS ${lib}) + endif() - add_custom_command(OUTPUT ${objlistfile} - COMMAND ${CMAKE_AR} -x "$" - COMMAND ${CMAKE_AR} -t "$" > ${objlistfile} - DEPENDS ${lib} ${objdir} - WORKING_DIRECTORY ${objdir}) + # LINUX: use "ar" to extract objects and re-add to a common lib + if(LINUX) + set(mri_file ${CMAKE_CURRENT_BINARY_DIR}/${TARGET_NAME}.mri CACHE INTERNAL "phi_static.mri file") + get_property(ABS_MERGE_LIB_PATH TARGET ${TARGET_NAME} PROPERTY LOCATION) + file(WRITE ${mri_file} "create ${ABS_MERGE_LIB_PATH}\n") - list(APPEND target_OBJS "${objlistfile}") + foreach(lib ${libs}) + get_property(ABS_LIB_PATH TARGET ${lib} PROPERTY LOCATION) + file(APPEND ${mri_file} "addlib ${ABS_LIB_PATH}\n") endforeach() - - # Make the generated dummy source file depended on all static input - # libs. If input lib changes,the source file is touched - # which causes the desired effect (relink). - add_custom_command(OUTPUT ${target_SRCS} - COMMAND ${CMAKE_COMMAND} -E touch ${target_SRCS} - DEPENDS ${libs} ${target_OBJS}) - - # Generate dummy staic lib - generate_dummy_static_lib(LIB_NAME ${TARGET_NAME} FILE_PATH ${target_SRCS} GENERATOR "generic.cmake:merge_static_libs") - - target_link_libraries(${TARGET_NAME} ${libs_deps}) - - # Get the file name of the generated library - set(target_LIBNAME "$") + file(APPEND ${mri_file} "save\nend\n") add_custom_command(TARGET ${TARGET_NAME} POST_BUILD - COMMAND ${CMAKE_AR} crs ${target_LIBNAME} `find ${target_DIR} -name '*.o'` - COMMAND ${CMAKE_RANLIB} ${target_LIBNAME} - WORKING_DIRECTORY ${target_DIR}) - endif(LINUX) - if(WIN32) # windows do not support gcc/nvcc combined compiling. Use msvc lib.exe to merge libs. - # Make the generated dummy source file depended on all static input - # libs. If input lib changes,the source file is touched - # which causes the desired effect (relink). - add_custom_command(OUTPUT ${target_SRCS} - COMMAND ${CMAKE_COMMAND} -E touch ${target_SRCS} - DEPENDS ${libs}) - # Generate dummy staic lib - generate_dummy_static_lib(LIB_NAME ${TARGET_NAME} FILE_PATH ${target_SRCS} GENERATOR "generic.cmake:merge_static_libs") - - target_link_libraries(${TARGET_NAME} ${libs_deps}) + COMMENT "Merge and generate static lib: lib${TARGET_NAME}.a" + COMMAND ${CMAKE_AR} -M < ${mri_file} + COMMAND ${CMAKE_RANLIB} "$") + endif() + # Windows do not support gcc/nvcc combined compiling. Use msvc 'lib.exe' to merge libs. + if(WIN32) foreach(lib ${libs}) - # Get the file names of the libraries to be merged set(libfiles ${libfiles} $) endforeach() - # msvc will put libarary in directory of "/Release/xxxlib" by default - # COMMAND cmake -E remove "${CMAKE_CURRENT_BINARY_DIR}/${CMAKE_BUILD_TYPE}/${TARGET_NAME}.lib" + # msvc compiler will put libarary in directory of "/Release/xxxlib" by default add_custom_command(TARGET ${TARGET_NAME} POST_BUILD + COMMENT "Merge and generate static lib: lib${TARGET_NAME}.lib" COMMAND cmake -E make_directory $ COMMAND lib /OUT:$ ${libfiles} ) - endif(WIN32) -endfunction(merge_static_libs) + endif() +endfunction() function(check_coverage_opt TARGET_NAME SRCS) if(WITH_COVERAGE AND WITH_INCREMENTAL_COVERAGE) @@ -1076,4 +1075,3 @@ function(math_library TARGET) cc_library(${TARGET} SRCS ${cc_srcs} DEPS ${math_library_DEPS} ${math_common_deps}) endif() endfunction() - diff --git a/cmake/third_party.cmake b/cmake/third_party.cmake index f8a841fecbc0a..c8ef4ad16ea9d 100755 --- a/cmake/third_party.cmake +++ b/cmake/third_party.cmake @@ -357,10 +357,8 @@ if (WITH_PSCORE) include(external/libmct) # download, build, install libmct list(APPEND third_party_deps extern_libmct) - if (WITH_HETERPS) - include(external/rocksdb) # download, build, install libmct - list(APPEND third_party_deps extern_rocksdb) - endif() + include(external/rocksdb) # download, build, install libmct + list(APPEND third_party_deps extern_rocksdb) endif() if(WITH_XBYAK) diff --git a/paddle/fluid/distributed/collective/CMakeLists.txt b/paddle/fluid/distributed/collective/CMakeLists.txt index b1bc8169da566..a59b48b76e057 100644 --- a/paddle/fluid/distributed/collective/CMakeLists.txt +++ b/paddle/fluid/distributed/collective/CMakeLists.txt @@ -1,21 +1,21 @@ -cc_library(processgroup SRCS ProcessGroup.cc DEPS phi phi_api eager_api) -cc_library(eager_reducer SRCS reducer.cc DEPS eager_api processgroup phi phi_api string_helper) +cc_library(processgroup SRCS ProcessGroup.cc DEPS phi_api eager_api) +cc_library(eager_reducer SRCS reducer.cc DEPS eager_api processgroup phi_api string_helper) cc_library(nccl_tool SRCS NCCLTools.cc DEPS place cuda_stream enforce collective_helper device_context) if (WITH_DISTRIBUTE) - cc_library(processgroup_gloo SRCS ProcessGroupGloo.cc DEPS phi phi_api eager_api gloo_wrapper) + cc_library(processgroup_gloo SRCS ProcessGroupGloo.cc DEPS phi_api eager_api gloo_wrapper) endif() if(WITH_NCCL) - cc_library(processgroup_nccl SRCS ProcessGroupNCCL.cc NCCLTools.cc Common.cc DEPS place cuda_stream enforce collective_helper device_context phi phi_api eager_api) + cc_library(processgroup_nccl SRCS ProcessGroupNCCL.cc NCCLTools.cc Common.cc DEPS place cuda_stream enforce collective_helper device_context phi_api eager_api) if (WITH_DISTRIBUTE AND WITH_PSCORE) - cc_library(processgroup_heter SRCS ProcessGroupHeter.cc NCCLTools.cc Common.cc DEPS place cuda_stream enforce collective_helper device_context phi phi_api eager_api) + cc_library(processgroup_heter SRCS ProcessGroupHeter.cc NCCLTools.cc Common.cc DEPS place cuda_stream enforce collective_helper device_context phi_api eager_api) endif() endif() if(WITH_ASCEND_CL) - cc_library(processgroup_hccl SRCS ProcessGroupHCCL.cc HCCLTools.cc Common.cc DEPS place npu_stream enforce collective_helper device_context phi phi_api eager_api) + cc_library(processgroup_hccl SRCS ProcessGroupHCCL.cc HCCLTools.cc Common.cc DEPS place npu_stream enforce collective_helper device_context phi_api eager_api) if (WITH_DISTRIBUTE AND WITH_PSCORE) - cc_library(processgroup_heter SRCS ProcessGroupHeter.cc HCCLTools.cc Common.cc DEPS place npu_stream enforce collective_helper device_context phi phi_api eager_api) + cc_library(processgroup_heter SRCS ProcessGroupHeter.cc HCCLTools.cc Common.cc DEPS place npu_stream enforce collective_helper device_context phi_api eager_api) endif() endif() diff --git a/paddle/fluid/distributed/collective/ProcessGroupHeter.cc b/paddle/fluid/distributed/collective/ProcessGroupHeter.cc old mode 100644 new mode 100755 index ef57bb5ba232c..ba5734208123e --- a/paddle/fluid/distributed/collective/ProcessGroupHeter.cc +++ b/paddle/fluid/distributed/collective/ProcessGroupHeter.cc @@ -116,7 +116,7 @@ std::shared_ptr ProcessGroupHeter::AllReduce( HeterClient* client_ = HeterClient::GetInstance({switch_endpoint_}, {}, 0).get(); auto dense_cpu_tensor = cpu_tensors[0]; - std::vector send_size; + std::vector send_size; send_size.push_back(dense_cpu_tensor.numel()); int ret = client_->Send( gid_, {dense_cpu_tensor.name()}, send_size, dense_cpu_tensor.data(), @@ -212,7 +212,7 @@ std::shared_ptr ProcessGroupHeter::Broadcast( HeterClient::GetInstance({switch_endpoint_}, {}, 0).get(); auto dense_cpu_tensor = cpu_tensors[0]; if (gloo_rank_ == 0) { - std::vector send_size; + std::vector send_size; send_size.push_back(dense_cpu_tensor.numel()); int ret = client_->Send( gid_, {dense_cpu_tensor.name()}, send_size, diff --git a/paddle/fluid/distributed/common/topk_calculator.h b/paddle/fluid/distributed/common/topk_calculator.h new file mode 100644 index 0000000000000..326f0f718e9bd --- /dev/null +++ b/paddle/fluid/distributed/common/topk_calculator.h @@ -0,0 +1,70 @@ +// Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once +#include +#include + +namespace paddle { +namespace distributed { +class TopkCalculator { + public: + TopkCalculator(int shard_num, size_t k) + : _shard_num(shard_num), _total_max_size(k) { + _shard_max_size = _total_max_size / shard_num; + _shard_max_size = _shard_max_size > 1 ? _shard_max_size : 1; + for (int i = 0; i < shard_num; ++i) { + _mpq.emplace(i, std::priority_queue, + std::greater>()); + } + } + ~TopkCalculator() {} + bool push(int shard_id, double value) { + if (_mpq.find(shard_id) == _mpq.end()) { + return false; + } + auto &pq = _mpq[shard_id]; + if (pq.size() < _shard_max_size) { + pq.push(value); + } else { + if (pq.top() < value) { + pq.pop(); + pq.push(value); + } + } + return true; + } + // TODO 再进行一次堆排序merge各个shard的结果 + int top() { + double total = 0; + for (const auto &item : _mpq) { + auto &pq = item.second; + if (!pq.empty()) { + total += pq.top(); + } + } + return total / _shard_num; + } + + private: + std::unordered_map, + std::greater>> + _mpq; + int _shard_num; + size_t _total_max_size; + size_t _shard_max_size; +}; + +} // namespace distributed +} // namespace paddle diff --git a/paddle/fluid/distributed/ps/service/CMakeLists.txt b/paddle/fluid/distributed/ps/service/CMakeLists.txt index b8de291072a1f..e7519ef4998b1 100755 --- a/paddle/fluid/distributed/ps/service/CMakeLists.txt +++ b/paddle/fluid/distributed/ps/service/CMakeLists.txt @@ -1,7 +1,16 @@ set(BRPC_SRCS ps_client.cc server.cc) set_source_files_properties(${BRPC_SRCS}) -set(BRPC_DEPS brpc ssl crypto protobuf gflags glog zlib leveldb snappy gflags glog device_context) + +if(WITH_HETERPS) + + set(BRPC_DEPS brpc ssl crypto protobuf gflags glog zlib leveldb snappy gflags glog device_context rocksdb) + +else() + + set(BRPC_DEPS brpc ssl crypto protobuf gflags glog zlib leveldb snappy gflags glog device_context) + +endif() brpc_library(sendrecv_rpc SRCS ${BRPC_SRCS} diff --git a/paddle/fluid/distributed/ps/service/brpc_ps_client.cc b/paddle/fluid/distributed/ps/service/brpc_ps_client.cc old mode 100644 new mode 100755 index 971c448bf2714..78673184eb23b --- a/paddle/fluid/distributed/ps/service/brpc_ps_client.cc +++ b/paddle/fluid/distributed/ps/service/brpc_ps_client.cc @@ -55,8 +55,6 @@ DEFINE_int32(pserver_sparse_merge_thread, 1, "pserver sparse merge thread num"); DEFINE_int32(pserver_sparse_table_shard_num, 1000, "sparse table shard for save & load"); -DEFINE_int32(heter_world_size, 100, "group size"); // 可配置 - namespace paddle { namespace framework { class Scope; @@ -429,6 +427,82 @@ std::future BrpcPsClient::Save(uint32_t table_id, return SendSaveCmd(table_id, PS_SAVE_ONE_TABLE, {epoch, mode}); } +std::future BrpcPsClient::CacheShuffle( + uint32_t table_id, const std::string &path, const std::string &mode, + const std::string &cache_threshold) { + VLOG(1) << "BrpcPsClient send cmd for cache shuffle"; + return SendSaveCmd(table_id, PS_CACHE_SHUFFLE, {path, mode, cache_threshold}); +} + +std::future BrpcPsClient::CacheShuffleMultiTable( + std::vector tables, const std::string &path, const std::string &mode, + const std::string &cache_threshold) { + VLOG(1) << "BrpcPsClient send cmd for cache shuffle multi table one path"; + std::vector param; + param.push_back(path); + param.push_back(mode); + param.push_back(cache_threshold); + for (size_t i = 0; i < tables.size(); i++) { + param.push_back(std::to_string(tables[i])); + } + return SendSaveCmd(0, PS_CACHE_SHUFFLE, param); +} + +std::future BrpcPsClient::SaveCache(uint32_t table_id, + const std::string &path, + const std::string &mode) { + return SendSaveCmd(table_id, PS_SAVE_ONE_CACHE_TABLE, {path, mode}); +} + +std::future BrpcPsClient::GetCacheThreshold(uint32_t table_id, + double &cache_threshold) { + int cmd_id = PS_GET_CACHE_THRESHOLD; + size_t request_call_num = _server_channels.size(); + DownpourBrpcClosure *closure = new DownpourBrpcClosure( + request_call_num, + [request_call_num, cmd_id, &cache_threshold](void *done) { + int ret = 0; + auto *closure = (DownpourBrpcClosure *)done; + std::vector cache_thresholds(request_call_num, 0); + for (size_t i = 0; i < request_call_num; ++i) { + if (closure->check_response(i, cmd_id) != 0) { + ret = -1; + break; + } + std::string cur_res = closure->get_response(i, cmd_id); + cache_thresholds[i] = std::stod(cur_res); + } + double sum_threshold = 0.0; + int count = 0; + for (auto t : cache_thresholds) { + if (t >= 0) { + sum_threshold += t; + ++count; + } + } + if (count == 0) { + cache_threshold = 0; + } else { + cache_threshold = sum_threshold / count; + } + VLOG(1) << "client get cache threshold: " << cache_threshold; + closure->set_promise_value(ret); + }); + auto promise = std::make_shared>(); + closure->add_promise(promise); + std::future fut = promise->get_future(); + for (size_t i = 0; i < request_call_num; ++i) { + closure->request(i)->set_cmd_id(cmd_id); + closure->request(i)->set_table_id(table_id); + closure->request(i)->set_client_id(_client_id); + PsService_Stub rpc_stub(GetCmdChannel(i)); + closure->cntl(i)->set_timeout_ms(10800000); + rpc_stub.service(closure->cntl(i), closure->request(i), + closure->response(i), closure); + } + return fut; +} + std::future BrpcPsClient::Clear() { return SendCmd(-1, PS_CLEAR_ALL_TABLE, {}); } diff --git a/paddle/fluid/distributed/ps/service/brpc_ps_client.h b/paddle/fluid/distributed/ps/service/brpc_ps_client.h index f109b473ca1f4..e2c16d496c42c 100644 --- a/paddle/fluid/distributed/ps/service/brpc_ps_client.h +++ b/paddle/fluid/distributed/ps/service/brpc_ps_client.h @@ -219,6 +219,20 @@ class BrpcPsClient : public PSClient { virtual int32_t RecvAndSaveTable(const uint64_t table_id, const std::string &path); + std::future CacheShuffle( + uint32_t table_id, const std::string &path, const std::string &mode, + const std::string &cache_threshold) override; + + std::future CacheShuffleMultiTable( + std::vector tables, const std::string &path, const std::string &mode, + const std::string &cache_threshold); + + std::future SaveCache(uint32_t table_id, const std::string &path, + const std::string &mode) override; + + std::future GetCacheThreshold(uint32_t table_id, + double &cache_threshold) override; + void PrintQueueSize(); void PrintQueueSizeThread(); diff --git a/paddle/fluid/distributed/ps/service/brpc_ps_server.cc b/paddle/fluid/distributed/ps/service/brpc_ps_server.cc index d22cca91f7816..d0bf06d49504a 100644 --- a/paddle/fluid/distributed/ps/service/brpc_ps_server.cc +++ b/paddle/fluid/distributed/ps/service/brpc_ps_server.cc @@ -28,6 +28,13 @@ class RpcController; } // namespace protobuf } // namespace google +DEFINE_int32(pserver_timeout_ms_s2s, 10000, + "pserver request server timeout_ms"); +DEFINE_int32(pserver_connect_timeout_ms_s2s, 10000, + "pserver connect server timeout_ms"); +DEFINE_string(pserver_connection_type_s2s, "pooled", + "pserver connection_type[pooled:single]"); + namespace paddle { namespace distributed { @@ -93,6 +100,84 @@ uint64_t BrpcPsServer::Start(const std::string &ip, uint32_t port) { return host.rank; } +int32_t BrpcPsServer::StartS2S() { + brpc::ChannelOptions options; + options.protocol = "baidu_std"; + options.timeout_ms = FLAGS_pserver_timeout_ms_s2s; + options.connection_type = FLAGS_pserver_connection_type_s2s; + options.connect_timeout_ms = FLAGS_pserver_connect_timeout_ms_s2s; + options.max_retry = 3; + + std::vector pserver_list = _environment->GetPsServers(); + _pserver_channels.resize(pserver_list.size()); + VLOG(2) << "pserver start s2s server_list size: " << _pserver_channels.size(); + + std::ostringstream os; + std::string server_ip_port; + + for (size_t i = 0; i < pserver_list.size(); ++i) { + server_ip_port.assign(pserver_list[i].ip.c_str()); + server_ip_port.append(":"); + server_ip_port.append(std::to_string(pserver_list[i].port)); + _pserver_channels[i].reset(new brpc::Channel()); + if (_pserver_channels[i]->Init(server_ip_port.c_str(), "", &options) != 0) { + LOG(ERROR) << "pserver connect to pserver:" << server_ip_port + << " Failed!"; + } + os << server_ip_port << ","; + } + LOG(INFO) << "pserver connect success: " << os.str(); + return 0; +} + +std::future BrpcPsServer::SendPServer2PServerMsg( + int msg_type, int to_pserver_id, const std::string &msg) { + auto promise = std::make_shared>(); + std::future fut = promise->get_future(); + if (to_pserver_id >= _pserver_channels.size()) { + LOG(FATAL) << "to_pserver_id is out of range pservers, which size is " + << _pserver_channels.size(); + promise->set_value(-1); + return fut; + } + auto *closure = new DownpourPServerBrpcClosure(1, [msg_type](void *done) { + auto *closure = (DownpourPServerBrpcClosure *)done; + int32_t ret = closure->check_response(0, msg_type + 1000); + closure->set_promise_value(ret); + }); + + closure->add_promise(promise); + closure->request(0)->set_cmd_id(101); + closure->request(0)->set_client_id(_rank); + closure->request(0)->set_table_id(0); + closure->request(0)->set_data(msg); + PsService_Stub rpc_stub(_pserver_channels[to_pserver_id].get()); + rpc_stub.service(closure->cntl(0), closure->request(0), closure->response(0), + closure); + return fut; +} + +int32_t BrpcPsServer::ReceiveFromPServer(int msg_type, int pserver_id, + const std::string &msg) { + if (msg.length() == 0) { + LOG(WARNING) << "SERVER>>RESPONSE>>msg = 0 Finish S2S Response"; + return 0; + } + paddle::framework::BinaryArchive ar; + ar.SetReadBuffer(const_cast(msg.c_str()), msg.length(), nullptr); + if (ar.Cursor() == ar.Finish()) { + LOG(WARNING) << "SERVER>>RESPONSE ar = 0>> Finish S2S Response"; + return 0; + } + std::vector> data; + while (ar.Cursor() < ar.Finish()) { + data.push_back(ar.Get>()); + } + CHECK(ar.Cursor() == ar.Finish()); + this->_shuffled_ins->Write(std::move(data)); + return 0; +} + int32_t BrpcPsServer::Port() { return _server.listen_address().port; } int32_t BrpcPsService::Initialize() { @@ -117,6 +202,14 @@ int32_t BrpcPsService::Initialize() { _service_handler_map[PS_START_PROFILER] = &BrpcPsService::StartProfiler; _service_handler_map[PS_STOP_PROFILER] = &BrpcPsService::StopProfiler; _service_handler_map[PS_PUSH_GLOBAL_STEP] = &BrpcPsService::PushGlobalStep; + // for save cache + + _service_handler_map[PS_SAVE_ONE_CACHE_TABLE] = + &BrpcPsService::SaveCacheTable; + _service_handler_map[PS_GET_CACHE_THRESHOLD] = + &BrpcPsService::GetCacheThreshold; + _service_handler_map[PS_CACHE_SHUFFLE] = &BrpcPsService::CacheShuffle; + auto &profiler = CostProfiler::instance(); profiler.register_profiler("pserver_server_pull_dense"); profiler.register_profiler("pserver_server_push_dense"); @@ -168,19 +261,29 @@ void BrpcPsService::service(google::protobuf::RpcController *cntl_base, response->set_err_msg(""); auto *table = _server->GetTable(request->table_id()); brpc::Controller *cntl = static_cast(cntl_base); - auto itr = _service_handler_map.find(request->cmd_id()); - if (itr == _service_handler_map.end()) { - std::string err_msg( - "undefined cmd_id, should match PsCmdID in ps.proto, cmd_id:"); - err_msg.append(std::to_string(request->cmd_id())); - set_response_code(*response, -1, err_msg.c_str()); - return; - } - serviceHandlerFunc handler_func = itr->second; - int service_ret = (this->*handler_func)(table, *request, *response, cntl); - if (service_ret != 0) { - response->set_err_code(service_ret); - response->set_err_msg("server internal error"); + + if (request->cmd_id() < 100) { + auto itr = _service_handler_map.find(request->cmd_id()); + if (itr == _service_handler_map.end()) { + std::string err_msg( + "undefined cmd_id, should match PsCmdID in ps.proto, cmd_id:"); + err_msg.append(std::to_string(request->cmd_id())); + set_response_code(*response, -1, err_msg.c_str()); + return; + } + serviceHandlerFunc handler_func = itr->second; + int service_ret = (this->*handler_func)(table, *request, *response, cntl); + if (service_ret != 0) { + response->set_err_code(service_ret); + response->set_err_msg("server internal error"); + } + } else { + int service_ret = _server->HandlePServer2PServerMsg( + request->cmd_id(), request->client_id(), request->data()); + if (service_ret != 0) { + response->set_err_code(-1); + response->set_err_msg("handle_pserver2pserver_msg failed"); + } } } @@ -561,6 +664,90 @@ int32_t BrpcPsService::SaveAllTable(Table *table, return 0; } +int32_t BrpcPsService::SaveCacheTable(Table *table, + const PsRequestMessage &request, + PsResponseMessage &response, + brpc::Controller *cntl) { + CHECK_TABLE_EXIST(table, request, response) + if (request.params_size() < 2) { + set_response_code( + response, -1, + "PsRequestMessage.datas is requeired at least 3, path&mode"); + return -1; + } + table->Flush(); + int32_t feasign_size = 0; + // if (_server->_shuffled_ins->size() <= 0) { + // LOG(WARNING) << "shuffled ins size <= 0"; + //} + feasign_size = table->SaveCache(request.params(0), request.params(1), + _server->_shuffled_ins); + if (feasign_size < 0) { + set_response_code(response, -1, "table save failed"); + return -1; + } + return feasign_size; +} + +int32_t BrpcPsService::CacheShuffle(Table *table, + const PsRequestMessage &request, + PsResponseMessage &response, + brpc::Controller *cntl) { + // start cache shuffle + CHECK_TABLE_EXIST(table, request, response) + if (request.params_size() < 3) { + set_response_code(response, -1, + "PsRequestMessage.datas is requeired at least 3, " + "path&mode&cache_threshold"); + return -1; + } + table->Flush(); + double cache_threshold = std::stod(request.params(2)); + LOG(INFO) << "cache threshold for cache shuffle: " << cache_threshold; + // auto shuffled_ins = paddle::ps::make_channel>(); + // shuffled_ins->set_block_size(80000); + _server->StartS2S(); + std::function(int msg_type, int to_pserver_id, + const std::string &msg)> + send_msg_func = [this](int msg_type, int to_pserver_id, + const std::string &msg) -> std::future { + return this->_server->SendPServer2PServerMsg(msg_type, to_pserver_id, msg); + }; + + std::vector table_ptrs; + for (size_t i = 3; i < request.params_size(); ++i) { + int table_id = std::stoi(request.params(i)); + Table *table_ptr = _server->GetTable(table_id); + table_ptrs.push_back(table_ptr); + } + if (table_ptrs.empty()) { + table_ptrs.push_back(table); + } + + table->CacheShuffle(request.params(0), request.params(1), cache_threshold, + send_msg_func, _server->_shuffled_ins, table_ptrs); + return 0; +} + +int32_t BrpcPsService::GetCacheThreshold(Table *table, + const PsRequestMessage &request, + PsResponseMessage &response, + brpc::Controller *cntl) { + CHECK_TABLE_EXIST(table, request, response) + table->Flush(); + double cache_threshold = 0.0; + cache_threshold = table->GetCacheThreshold(); + if (cache_threshold < 0) { + LOG(WARNING) << "wrong threshold: " << cache_threshold; + } + std::stringstream ss; + ss << std::setprecision(15) << cache_threshold; + std::string cache_threshold_str = ss.str(); + response.set_data(cache_threshold_str); + return 0; +} + int32_t BrpcPsService::ShrinkTable(Table *table, const PsRequestMessage &request, PsResponseMessage &response, diff --git a/paddle/fluid/distributed/ps/service/brpc_ps_server.h b/paddle/fluid/distributed/ps/service/brpc_ps_server.h index 250f465d84253..40ed652ec6be3 100644 --- a/paddle/fluid/distributed/ps/service/brpc_ps_server.h +++ b/paddle/fluid/distributed/ps/service/brpc_ps_server.h @@ -53,6 +53,12 @@ class BrpcPsServer : public PSServer { } int32_t Port(); + virtual int32_t StartS2S() override; + virtual ::std::future SendPServer2PServerMsg( + int msg_type, int to_pserver_id, const std::string &msg) override; + virtual int32_t ReceiveFromPServer(int msg_type, int pserver_id, + const std::string &msg) override; + private: virtual int32_t Initialize(); mutable std::mutex mutex_; @@ -123,6 +129,16 @@ class BrpcPsService : public PsBaseService { int32_t PushGlobalStep(Table *table, const PsRequestMessage &request, PsResponseMessage &response, brpc::Controller *cntl); + int32_t CacheShuffle(Table *table, const PsRequestMessage &request, + PsResponseMessage &response, brpc::Controller *cntl); + + int32_t SaveCacheTable(Table *table, const PsRequestMessage &request, + PsResponseMessage &response, brpc::Controller *cntl); + + int32_t GetCacheThreshold(Table *table, const PsRequestMessage &request, + PsResponseMessage &response, + brpc::Controller *cntl); + bool _is_initialize_shard_info; std::mutex _initialize_shard_mutex; std::unordered_map _service_handler_map; diff --git a/paddle/fluid/distributed/ps/service/graph_brpc_client.cc b/paddle/fluid/distributed/ps/service/graph_brpc_client.cc index 827a643ee50d6..c1df490669dbe 100644 --- a/paddle/fluid/distributed/ps/service/graph_brpc_client.cc +++ b/paddle/fluid/distributed/ps/service/graph_brpc_client.cc @@ -53,7 +53,7 @@ int GraphBrpcClient::get_server_index_by_id(int64_t id) { } std::future GraphBrpcClient::get_node_feat( - const uint32_t &table_id, const std::vector &node_ids, + const uint32_t &table_id, int idx_, const std::vector &node_ids, const std::vector &feature_names, std::vector> &res) { std::vector request2server; @@ -124,9 +124,11 @@ std::future GraphBrpcClient::get_node_feat( int server_index = request2server[request_idx]; closure->request(request_idx)->set_cmd_id(PS_GRAPH_GET_NODE_FEAT); closure->request(request_idx)->set_table_id(table_id); + closure->request(request_idx)->set_client_id(_client_id); size_t node_num = node_id_buckets[request_idx].size(); + closure->request(request_idx)->add_params((char *)&idx_, sizeof(int)); closure->request(request_idx) ->add_params((char *)node_id_buckets[request_idx].data(), sizeof(int64_t) * node_num); @@ -144,7 +146,8 @@ std::future GraphBrpcClient::get_node_feat( return fut; } -std::future GraphBrpcClient::clear_nodes(uint32_t table_id) { +std::future GraphBrpcClient::clear_nodes(uint32_t table_id, + int type_id, int idx_) { DownpourBrpcClosure *closure = new DownpourBrpcClosure( server_size, [&, server_size = this->server_size ](void *done) { int ret = 0; @@ -167,7 +170,8 @@ std::future GraphBrpcClient::clear_nodes(uint32_t table_id) { closure->request(server_index)->set_cmd_id(PS_GRAPH_CLEAR); closure->request(server_index)->set_table_id(table_id); closure->request(server_index)->set_client_id(_client_id); - + closure->request(server_index)->add_params((char *)&type_id, sizeof(int)); + closure->request(server_index)->add_params((char *)&idx_, sizeof(int)); GraphPsService_Stub rpc_stub = getServiceStub(GetCmdChannel(server_index)); closure->cntl(server_index)->set_log_id(butil::gettimeofday_ms()); rpc_stub.service(closure->cntl(server_index), @@ -177,7 +181,7 @@ std::future GraphBrpcClient::clear_nodes(uint32_t table_id) { return fut; } std::future GraphBrpcClient::add_graph_node( - uint32_t table_id, std::vector &node_id_list, + uint32_t table_id, int idx_, std::vector &node_id_list, std::vector &is_weighted_list) { std::vector> request_bucket; std::vector> is_weighted_bucket; @@ -225,6 +229,7 @@ std::future GraphBrpcClient::add_graph_node( closure->request(request_idx)->set_table_id(table_id); closure->request(request_idx)->set_client_id(_client_id); size_t node_num = request_bucket[request_idx].size(); + closure->request(request_idx)->add_params((char *)&idx_, sizeof(int)); closure->request(request_idx) ->add_params((char *)request_bucket[request_idx].data(), sizeof(int64_t) * node_num); @@ -245,7 +250,7 @@ std::future GraphBrpcClient::add_graph_node( return fut; } std::future GraphBrpcClient::remove_graph_node( - uint32_t table_id, std::vector &node_id_list) { + uint32_t table_id, int idx_, std::vector &node_id_list) { std::vector> request_bucket; std::vector server_index_arr; std::vector index_mapping(server_size, -1); @@ -286,6 +291,7 @@ std::future GraphBrpcClient::remove_graph_node( closure->request(request_idx)->set_client_id(_client_id); size_t node_num = request_bucket[request_idx].size(); + closure->request(request_idx)->add_params((char *)&idx_, sizeof(int)); closure->request(request_idx) ->add_params((char *)request_bucket[request_idx].data(), sizeof(int64_t) * node_num); @@ -299,7 +305,7 @@ std::future GraphBrpcClient::remove_graph_node( } // char* &buffer,int &actual_size std::future GraphBrpcClient::batch_sample_neighbors( - uint32_t table_id, std::vector node_ids, int sample_size, + uint32_t table_id, int idx_, std::vector node_ids, int sample_size, // std::vector>> &res, std::vector> &res, std::vector> &res_weight, bool need_weight, @@ -353,6 +359,7 @@ std::future GraphBrpcClient::batch_sample_neighbors( closure->request(0)->set_cmd_id(PS_GRAPH_SAMPLE_NODES_FROM_ONE_SERVER); closure->request(0)->set_table_id(table_id); closure->request(0)->set_client_id(_client_id); + closure->request(0)->add_params((char *)&idx_, sizeof(int)); closure->request(0)->add_params((char *)node_ids.data(), sizeof(int64_t) * node_ids.size()); closure->request(0)->add_params((char *)&sample_size, sizeof(int)); @@ -452,6 +459,7 @@ std::future GraphBrpcClient::batch_sample_neighbors( closure->request(request_idx)->set_client_id(_client_id); size_t node_num = node_id_buckets[request_idx].size(); + closure->request(request_idx)->add_params((char *)&idx_, sizeof(int)); closure->request(request_idx) ->add_params((char *)node_id_buckets[request_idx].data(), sizeof(int64_t) * node_num); @@ -469,7 +477,7 @@ std::future GraphBrpcClient::batch_sample_neighbors( return fut; } std::future GraphBrpcClient::random_sample_nodes( - uint32_t table_id, int server_index, int sample_size, + uint32_t table_id, int type_id, int idx_, int server_index, int sample_size, std::vector &ids) { DownpourBrpcClosure *closure = new DownpourBrpcClosure(1, [&](void *done) { int ret = 0; @@ -498,6 +506,8 @@ std::future GraphBrpcClient::random_sample_nodes( closure->request(0)->set_cmd_id(PS_GRAPH_SAMPLE_NODES); closure->request(0)->set_table_id(table_id); closure->request(0)->set_client_id(_client_id); + closure->request(0)->add_params((char *)&type_id, sizeof(int)); + closure->request(0)->add_params((char *)&idx_, sizeof(int)); closure->request(0)->add_params((char *)&sample_size, sizeof(int)); ; // PsService_Stub rpc_stub(GetCmdChannel(server_index)); @@ -508,83 +518,9 @@ std::future GraphBrpcClient::random_sample_nodes( return fut; } -std::future GraphBrpcClient::load_graph_split_config( - uint32_t table_id, std::string path) { - DownpourBrpcClosure *closure = new DownpourBrpcClosure( - server_size, [&, server_size = this->server_size ](void *done) { - int ret = 0; - auto *closure = (DownpourBrpcClosure *)done; - size_t fail_num = 0; - for (size_t request_idx = 0; request_idx < server_size; ++request_idx) { - if (closure->check_response(request_idx, - PS_GRAPH_LOAD_GRAPH_SPLIT_CONFIG) != 0) { - ++fail_num; - break; - } - } - ret = fail_num == 0 ? 0 : -1; - closure->set_promise_value(ret); - }); - auto promise = std::make_shared>(); - closure->add_promise(promise); - std::future fut = promise->get_future(); - for (size_t i = 0; i < server_size; i++) { - int server_index = i; - closure->request(server_index) - ->set_cmd_id(PS_GRAPH_LOAD_GRAPH_SPLIT_CONFIG); - closure->request(server_index)->set_table_id(table_id); - closure->request(server_index)->set_client_id(_client_id); - closure->request(server_index)->add_params(path); - GraphPsService_Stub rpc_stub = getServiceStub(GetCmdChannel(server_index)); - closure->cntl(server_index)->set_log_id(butil::gettimeofday_ms()); - rpc_stub.service(closure->cntl(server_index), - closure->request(server_index), - closure->response(server_index), closure); - } - return fut; -} -std::future GraphBrpcClient::use_neighbors_sample_cache( - uint32_t table_id, size_t total_size_limit, size_t ttl) { - DownpourBrpcClosure *closure = new DownpourBrpcClosure( - server_size, [&, server_size = this->server_size ](void *done) { - int ret = 0; - auto *closure = (DownpourBrpcClosure *)done; - size_t fail_num = 0; - for (size_t request_idx = 0; request_idx < server_size; ++request_idx) { - if (closure->check_response( - request_idx, PS_GRAPH_USE_NEIGHBORS_SAMPLE_CACHE) != 0) { - ++fail_num; - break; - } - } - ret = fail_num == 0 ? 0 : -1; - closure->set_promise_value(ret); - }); - auto promise = std::make_shared>(); - closure->add_promise(promise); - size_t size_limit = total_size_limit / server_size + - (total_size_limit % server_size != 0 ? 1 : 0); - std::future fut = promise->get_future(); - for (size_t i = 0; i < server_size; i++) { - int server_index = i; - closure->request(server_index) - ->set_cmd_id(PS_GRAPH_USE_NEIGHBORS_SAMPLE_CACHE); - closure->request(server_index)->set_table_id(table_id); - closure->request(server_index)->set_client_id(_client_id); - closure->request(server_index) - ->add_params((char *)&size_limit, sizeof(size_t)); - closure->request(server_index)->add_params((char *)&ttl, sizeof(size_t)); - GraphPsService_Stub rpc_stub = getServiceStub(GetCmdChannel(server_index)); - closure->cntl(server_index)->set_log_id(butil::gettimeofday_ms()); - rpc_stub.service(closure->cntl(server_index), - closure->request(server_index), - closure->response(server_index), closure); - } - return fut; -} std::future GraphBrpcClient::pull_graph_list( - uint32_t table_id, int server_index, int start, int size, int step, - std::vector &res) { + uint32_t table_id, int type_id, int idx_, int server_index, int start, + int size, int step, std::vector &res) { DownpourBrpcClosure *closure = new DownpourBrpcClosure(1, [&](void *done) { int ret = 0; auto *closure = (DownpourBrpcClosure *)done; @@ -613,6 +549,8 @@ std::future GraphBrpcClient::pull_graph_list( closure->request(0)->set_cmd_id(PS_PULL_GRAPH_LIST); closure->request(0)->set_table_id(table_id); closure->request(0)->set_client_id(_client_id); + closure->request(0)->add_params((char *)&type_id, sizeof(int)); + closure->request(0)->add_params((char *)&idx_, sizeof(int)); closure->request(0)->add_params((char *)&start, sizeof(int)); closure->request(0)->add_params((char *)&size, sizeof(int)); closure->request(0)->add_params((char *)&step, sizeof(int)); @@ -625,7 +563,7 @@ std::future GraphBrpcClient::pull_graph_list( } std::future GraphBrpcClient::set_node_feat( - const uint32_t &table_id, const std::vector &node_ids, + const uint32_t &table_id, int idx_, const std::vector &node_ids, const std::vector &feature_names, const std::vector> &features) { std::vector request2server; @@ -686,6 +624,7 @@ std::future GraphBrpcClient::set_node_feat( closure->request(request_idx)->set_client_id(_client_id); size_t node_num = node_id_buckets[request_idx].size(); + closure->request(request_idx)->add_params((char *)&idx_, sizeof(int)); closure->request(request_idx) ->add_params((char *)node_id_buckets[request_idx].data(), sizeof(int64_t) * node_num); diff --git a/paddle/fluid/distributed/ps/service/graph_brpc_client.h b/paddle/fluid/distributed/ps/service/graph_brpc_client.h index d1d3c95260df4..51f14bc57cde0 100644 --- a/paddle/fluid/distributed/ps/service/graph_brpc_client.h +++ b/paddle/fluid/distributed/ps/service/graph_brpc_client.h @@ -63,40 +63,37 @@ class GraphBrpcClient : public BrpcPsClient { virtual ~GraphBrpcClient() {} // given a batch of nodes, sample graph_neighbors for each of them virtual std::future batch_sample_neighbors( - uint32_t table_id, std::vector node_ids, int sample_size, - std::vector>& res, + uint32_t table_id, int idx, std::vector node_ids, + int sample_size, std::vector>& res, std::vector>& res_weight, bool need_weight, int server_index = -1); - virtual std::future pull_graph_list(uint32_t table_id, - int server_index, int start, - int size, int step, + virtual std::future pull_graph_list(uint32_t table_id, int type_id, + int idx, int server_index, + int start, int size, int step, std::vector& res); virtual std::future random_sample_nodes(uint32_t table_id, + int type_id, int idx, int server_index, int sample_size, std::vector& ids); virtual std::future get_node_feat( - const uint32_t& table_id, const std::vector& node_ids, + const uint32_t& table_id, int idx, const std::vector& node_ids, const std::vector& feature_names, std::vector>& res); virtual std::future set_node_feat( - const uint32_t& table_id, const std::vector& node_ids, + const uint32_t& table_id, int idx, const std::vector& node_ids, const std::vector& feature_names, const std::vector>& features); - virtual std::future clear_nodes(uint32_t table_id); + virtual std::future clear_nodes(uint32_t table_id, int type_id, + int idx); virtual std::future add_graph_node( - uint32_t table_id, std::vector& node_id_list, + uint32_t table_id, int idx, std::vector& node_id_list, std::vector& is_weighted_list); - virtual std::future use_neighbors_sample_cache(uint32_t table_id, - size_t size_limit, - size_t ttl); - virtual std::future load_graph_split_config(uint32_t table_id, - std::string path); virtual std::future remove_graph_node( - uint32_t table_id, std::vector& node_id_list); + uint32_t table_id, int idx_, std::vector& node_id_list); virtual int32_t Initialize(); int get_shard_num() { return shard_num; } void set_shard_num(int shard_num) { this->shard_num = shard_num; } diff --git a/paddle/fluid/distributed/ps/service/graph_brpc_server.cc b/paddle/fluid/distributed/ps/service/graph_brpc_server.cc index 21e590997b178..8ff12265269b2 100644 --- a/paddle/fluid/distributed/ps/service/graph_brpc_server.cc +++ b/paddle/fluid/distributed/ps/service/graph_brpc_server.cc @@ -124,7 +124,9 @@ int32_t GraphBrpcService::clear_nodes(Table *table, const PsRequestMessage &request, PsResponseMessage &response, brpc::Controller *cntl) { - ((GraphTable *)table)->clear_nodes(); + int type_id = *(int *)(request.params(0).c_str()); + int idx_ = *(int *)(request.params(1).c_str()); + ((GraphTable *)table)->clear_nodes(type_id, idx_); return 0; } @@ -133,25 +135,34 @@ int32_t GraphBrpcService::add_graph_node(Table *table, PsResponseMessage &response, brpc::Controller *cntl) { CHECK_TABLE_EXIST(table, request, response) - if (request.params_size() < 1) { - set_response_code( - response, -1, - "graph_get_node_feat request requires at least 2 arguments"); + if (request.params_size() < 2) { + set_response_code(response, -1, + "add_graph_node request requires at least 2 arguments"); return 0; } - size_t node_num = request.params(0).size() / sizeof(int64_t); - int64_t *node_data = (int64_t *)(request.params(0).c_str()); + int idx_ = *(int *)(request.params(0).c_str()); + size_t node_num = request.params(1).size() / sizeof(int64_t); + int64_t *node_data = (int64_t *)(request.params(1).c_str()); + // size_t node_num = request.params(0).size() / sizeof(int64_t); + // int64_t *node_data = (int64_t *)(request.params(0).c_str()); std::vector node_ids(node_data, node_data + node_num); std::vector is_weighted_list; - if (request.params_size() == 2) { - size_t weight_list_size = request.params(1).size() / sizeof(bool); - bool *is_weighted_buffer = (bool *)(request.params(1).c_str()); + if (request.params_size() == 3) { + size_t weight_list_size = request.params(2).size() / sizeof(bool); + bool *is_weighted_buffer = (bool *)(request.params(2).c_str()); is_weighted_list = std::vector(is_weighted_buffer, is_weighted_buffer + weight_list_size); } + // if (request.params_size() == 2) { + // size_t weight_list_size = request.params(1).size() / sizeof(bool); + // bool *is_weighted_buffer = (bool *)(request.params(1).c_str()); + // is_weighted_list = std::vector(is_weighted_buffer, + // is_weighted_buffer + + // weight_list_size); + // } - ((GraphTable *)table)->add_graph_node(node_ids, is_weighted_list); + ((GraphTable *)table)->add_graph_node(idx_, node_ids, is_weighted_list); return 0; } int32_t GraphBrpcService::remove_graph_node(Table *table, @@ -159,17 +170,20 @@ int32_t GraphBrpcService::remove_graph_node(Table *table, PsResponseMessage &response, brpc::Controller *cntl) { CHECK_TABLE_EXIST(table, request, response) - if (request.params_size() < 1) { + if (request.params_size() < 2) { set_response_code( response, -1, - "graph_get_node_feat request requires at least 1 argument"); + "remove_graph_node request requires at least 2 arguments"); return 0; } - size_t node_num = request.params(0).size() / sizeof(int64_t); - int64_t *node_data = (int64_t *)(request.params(0).c_str()); + int idx_ = *(int *)(request.params(0).c_str()); + size_t node_num = request.params(1).size() / sizeof(int64_t); + int64_t *node_data = (int64_t *)(request.params(1).c_str()); + // size_t node_num = request.params(0).size() / sizeof(int64_t); + // int64_t *node_data = (int64_t *)(request.params(0).c_str()); std::vector node_ids(node_data, node_data + node_num); - ((GraphTable *)table)->remove_graph_node(node_ids); + ((GraphTable *)table)->remove_graph_node(idx_, node_ids); return 0; } int32_t GraphBrpcServer::Port() { return _server.listen_address().port; } @@ -201,10 +215,10 @@ int32_t GraphBrpcService::Initialize() { &GraphBrpcService::graph_set_node_feat; _service_handler_map[PS_GRAPH_SAMPLE_NODES_FROM_ONE_SERVER] = &GraphBrpcService::sample_neighbors_across_multi_servers; - _service_handler_map[PS_GRAPH_USE_NEIGHBORS_SAMPLE_CACHE] = - &GraphBrpcService::use_neighbors_sample_cache; - _service_handler_map[PS_GRAPH_LOAD_GRAPH_SPLIT_CONFIG] = - &GraphBrpcService::load_graph_split_config; + // _service_handler_map[PS_GRAPH_USE_NEIGHBORS_SAMPLE_CACHE] = + // &GraphBrpcService::use_neighbors_sample_cache; + // _service_handler_map[PS_GRAPH_LOAD_GRAPH_SPLIT_CONFIG] = + // &GraphBrpcService::load_graph_split_config; // shard初始化,server启动后才可从env获取到server_list的shard信息 InitializeShardInfo(); @@ -360,18 +374,24 @@ int32_t GraphBrpcService::pull_graph_list(Table *table, PsResponseMessage &response, brpc::Controller *cntl) { CHECK_TABLE_EXIST(table, request, response) - if (request.params_size() < 3) { + if (request.params_size() < 5) { set_response_code(response, -1, - "pull_graph_list request requires at least 3 arguments"); + "pull_graph_list request requires at least 5 arguments"); return 0; } - int start = *(int *)(request.params(0).c_str()); - int size = *(int *)(request.params(1).c_str()); - int step = *(int *)(request.params(2).c_str()); + int type_id = *(int *)(request.params(0).c_str()); + int idx = *(int *)(request.params(1).c_str()); + int start = *(int *)(request.params(2).c_str()); + int size = *(int *)(request.params(3).c_str()); + int step = *(int *)(request.params(4).c_str()); + // int start = *(int *)(request.params(0).c_str()); + // int size = *(int *)(request.params(1).c_str()); + // int step = *(int *)(request.params(2).c_str()); std::unique_ptr buffer; int actual_size; ((GraphTable *)table) - ->pull_graph_list(start, size, buffer, actual_size, false, step); + ->pull_graph_list(type_id, idx, start, size, buffer, actual_size, false, + step); cntl->response_attachment().append(buffer.get(), actual_size); return 0; } @@ -379,21 +399,26 @@ int32_t GraphBrpcService::graph_random_sample_neighbors( Table *table, const PsRequestMessage &request, PsResponseMessage &response, brpc::Controller *cntl) { CHECK_TABLE_EXIST(table, request, response) - if (request.params_size() < 3) { + if (request.params_size() < 4) { set_response_code( response, -1, "graph_random_sample_neighbors request requires at least 3 arguments"); return 0; } - size_t node_num = request.params(0).size() / sizeof(int64_t); - int64_t *node_data = (int64_t *)(request.params(0).c_str()); - int sample_size = *(int64_t *)(request.params(1).c_str()); - bool need_weight = *(bool *)(request.params(2).c_str()); + int idx_ = *(int *)(request.params(0).c_str()); + size_t node_num = request.params(1).size() / sizeof(int64_t); + int64_t *node_data = (int64_t *)(request.params(1).c_str()); + int sample_size = *(int64_t *)(request.params(2).c_str()); + bool need_weight = *(bool *)(request.params(3).c_str()); + // size_t node_num = request.params(0).size() / sizeof(int64_t); + // int64_t *node_data = (int64_t *)(request.params(0).c_str()); + // int sample_size = *(int64_t *)(request.params(1).c_str()); + // bool need_weight = *(bool *)(request.params(2).c_str()); std::vector> buffers(node_num); std::vector actual_sizes(node_num, 0); ((GraphTable *)table) - ->random_sample_neighbors(node_data, sample_size, buffers, actual_sizes, - need_weight); + ->random_sample_neighbors(idx_, node_data, sample_size, buffers, + actual_sizes, need_weight); cntl->response_attachment().append(&node_num, sizeof(size_t)); cntl->response_attachment().append(actual_sizes.data(), @@ -406,10 +431,14 @@ int32_t GraphBrpcService::graph_random_sample_neighbors( int32_t GraphBrpcService::graph_random_sample_nodes( Table *table, const PsRequestMessage &request, PsResponseMessage &response, brpc::Controller *cntl) { - size_t size = *(int64_t *)(request.params(0).c_str()); + int type_id = *(int *)(request.params(0).c_str()); + int idx_ = *(int *)(request.params(1).c_str()); + size_t size = *(int64_t *)(request.params(2).c_str()); + // size_t size = *(int64_t *)(request.params(0).c_str()); std::unique_ptr buffer; int actual_size; - if (((GraphTable *)table)->random_sample_nodes(size, buffer, actual_size) == + if (((GraphTable *)table) + ->random_sample_nodes(type_id, idx_, size, buffer, actual_size) == 0) { cntl->response_attachment().append(buffer.get(), actual_size); } else @@ -423,23 +452,26 @@ int32_t GraphBrpcService::graph_get_node_feat(Table *table, PsResponseMessage &response, brpc::Controller *cntl) { CHECK_TABLE_EXIST(table, request, response) - if (request.params_size() < 2) { + if (request.params_size() < 3) { set_response_code( response, -1, - "graph_get_node_feat request requires at least 2 arguments"); + "graph_get_node_feat request requires at least 3 arguments"); return 0; } - size_t node_num = request.params(0).size() / sizeof(int64_t); - int64_t *node_data = (int64_t *)(request.params(0).c_str()); + int idx_ = *(int *)(request.params(0).c_str()); + size_t node_num = request.params(1).size() / sizeof(int64_t); + int64_t *node_data = (int64_t *)(request.params(1).c_str()); + // size_t node_num = request.params(0).size() / sizeof(int64_t); + // int64_t *node_data = (int64_t *)(request.params(0).c_str()); std::vector node_ids(node_data, node_data + node_num); std::vector feature_names = - paddle::string::split_string(request.params(1), "\t"); + paddle::string::split_string(request.params(2), "\t"); std::vector> feature( feature_names.size(), std::vector(node_num)); - ((GraphTable *)table)->get_node_feat(node_ids, feature_names, feature); + ((GraphTable *)table)->get_node_feat(idx_, node_ids, feature_names, feature); for (size_t feat_idx = 0; feat_idx < feature_names.size(); ++feat_idx) { for (size_t node_idx = 0; node_idx < node_num; ++node_idx) { @@ -457,17 +489,25 @@ int32_t GraphBrpcService::sample_neighbors_across_multi_servers( brpc::Controller *cntl) { // sleep(5); CHECK_TABLE_EXIST(table, request, response) - if (request.params_size() < 3) { + if (request.params_size() < 4) { set_response_code(response, -1, "sample_neighbors_across_multi_servers request requires " - "at least 3 arguments"); + "at least 4 arguments"); return 0; } - size_t node_num = request.params(0).size() / sizeof(int64_t), + + int idx_ = *(int *)(request.params(0).c_str()); + size_t node_num = request.params(1).size() / sizeof(int64_t), size_of_size_t = sizeof(size_t); - int64_t *node_data = (int64_t *)(request.params(0).c_str()); - int sample_size = *(int64_t *)(request.params(1).c_str()); - bool need_weight = *(int64_t *)(request.params(2).c_str()); + int64_t *node_data = (int64_t *)(request.params(1).c_str()); + int sample_size = *(int64_t *)(request.params(2).c_str()); + bool need_weight = *(int64_t *)(request.params(3).c_str()); + + // size_t node_num = request.params(0).size() / sizeof(int64_t), + // size_of_size_t = sizeof(size_t); + // int64_t *node_data = (int64_t *)(request.params(0).c_str()); + // int sample_size = *(int64_t *)(request.params(1).c_str()); + // bool need_weight = *(int64_t *)(request.params(2).c_str()); // std::vector res = ((GraphTable // *)table).filter_out_non_exist_nodes(node_data, sample_size); std::vector request2server; @@ -580,6 +620,8 @@ int32_t GraphBrpcService::sample_neighbors_across_multi_servers( closure->request(request_idx)->set_client_id(rank); size_t node_num = node_id_buckets[request_idx].size(); + closure->request(request_idx)->add_params((char *)&idx_, sizeof(int)); + closure->request(request_idx) ->add_params((char *)node_id_buckets[request_idx].data(), sizeof(int64_t) * node_num); @@ -597,9 +639,9 @@ int32_t GraphBrpcService::sample_neighbors_across_multi_servers( } if (server2request[rank] != -1) { ((GraphTable *)table) - ->random_sample_neighbors(node_id_buckets.back().data(), sample_size, - local_buffers, local_actual_sizes, - need_weight); + ->random_sample_neighbors(idx_, node_id_buckets.back().data(), + sample_size, local_buffers, + local_actual_sizes, need_weight); } local_promise.get()->set_value(0); if (remote_call_num == 0) func(closure); @@ -611,23 +653,31 @@ int32_t GraphBrpcService::graph_set_node_feat(Table *table, PsResponseMessage &response, brpc::Controller *cntl) { CHECK_TABLE_EXIST(table, request, response) - if (request.params_size() < 3) { + if (request.params_size() < 4) { set_response_code( response, -1, "graph_set_node_feat request requires at least 3 arguments"); return 0; } - size_t node_num = request.params(0).size() / sizeof(int64_t); - int64_t *node_data = (int64_t *)(request.params(0).c_str()); + int idx_ = *(int *)(request.params(0).c_str()); + + // size_t node_num = request.params(0).size() / sizeof(int64_t); + // int64_t *node_data = (int64_t *)(request.params(0).c_str()); + size_t node_num = request.params(1).size() / sizeof(int64_t); + int64_t *node_data = (int64_t *)(request.params(1).c_str()); std::vector node_ids(node_data, node_data + node_num); + // std::vector feature_names = + // paddle::string::split_string(request.params(1), "\t"); + std::vector feature_names = - paddle::string::split_string(request.params(1), "\t"); + paddle::string::split_string(request.params(2), "\t"); std::vector> features( feature_names.size(), std::vector(node_num)); - const char *buffer = request.params(2).c_str(); + // const char *buffer = request.params(2).c_str(); + const char *buffer = request.params(3).c_str(); for (size_t feat_idx = 0; feat_idx < feature_names.size(); ++feat_idx) { for (size_t node_idx = 0; node_idx < node_num; ++node_idx) { @@ -639,40 +689,10 @@ int32_t GraphBrpcService::graph_set_node_feat(Table *table, } } - ((GraphTable *)table)->set_node_feat(node_ids, feature_names, features); + ((GraphTable *)table)->set_node_feat(idx_, node_ids, feature_names, features); return 0; } -int32_t GraphBrpcService::use_neighbors_sample_cache( - Table *table, const PsRequestMessage &request, PsResponseMessage &response, - brpc::Controller *cntl) { - CHECK_TABLE_EXIST(table, request, response) - if (request.params_size() < 2) { - set_response_code(response, -1, - "use_neighbors_sample_cache request requires at least 2 " - "arguments[cache_size, ttl]"); - return 0; - } - size_t size_limit = *(size_t *)(request.params(0).c_str()); - size_t ttl = *(size_t *)(request.params(1).c_str()); - ((GraphTable *)table)->make_neighbor_sample_cache(size_limit, ttl); - return 0; -} - -int32_t GraphBrpcService::load_graph_split_config( - Table *table, const PsRequestMessage &request, PsResponseMessage &response, - brpc::Controller *cntl) { - CHECK_TABLE_EXIST(table, request, response) - if (request.params_size() < 1) { - set_response_code(response, -1, - "load_graph_split_configrequest requires at least 1 " - "argument1[file_path]"); - return 0; - } - ((GraphTable *)table)->load_graph_split_config(request.params(0)); - return 0; -} - } // namespace distributed } // namespace paddle diff --git a/paddle/fluid/distributed/ps/service/heter_client.cc b/paddle/fluid/distributed/ps/service/heter_client.cc old mode 100644 new mode 100755 index 16c1ff764dc3c..8085ef68e1cad --- a/paddle/fluid/distributed/ps/service/heter_client.cc +++ b/paddle/fluid/distributed/ps/service/heter_client.cc @@ -17,9 +17,11 @@ #include "paddle/fluid/framework/convert_utils.h" #include "paddle/fluid/platform/profiler.h" +DEFINE_int32(heter_world_size, 100, "group size"); // group max size +DEFINE_int32(switch_send_recv_timeout_s, 600, "switch_send_recv_timeout_s"); + namespace paddle { namespace distributed { - std::shared_ptr HeterClient::s_instance_ = nullptr; int GetMicroId(const platform::DeviceContext& ctx, @@ -222,6 +224,7 @@ int HeterClient::Send(const platform::DeviceContext& ctx, distributed::MultiVarMsg request; // 1. set req message_name(string) request.set_message_name(message_name); + request.set_group_id(0); // 2. set req send_var_names() for (auto& send_var_name : send_var_names) { @@ -263,7 +266,7 @@ int HeterClient::Send(const platform::DeviceContext& ctx, } int HeterClient::Send(int group_id, const std::vector& var_names, - const std::vector& vars_len, void* data_ptr, + const std::vector& vars_size, void* data_ptr, int64_t data_size) { OnHeterRpcDone* closure = new OnHeterRpcDone([](void* done) { auto* closure = reinterpret_cast(done); @@ -282,7 +285,7 @@ int HeterClient::Send(int group_id, const std::vector& var_names, for (auto& send_var_name : var_names) { request.add_send_var_names(send_var_name); } - for (auto var_len : vars_len) { + for (auto var_len : vars_size) { request.add_vars_len(var_len); } auto& request_buffer = closure->cntl.request_attachment(); @@ -301,6 +304,7 @@ int HeterClient::Send(int group_id, const std::vector& var_names, ::paddle::distributed::PsService_Stub stub(channel); stub.SendToSwitch(&closure->cntl, &request, &closure->ps_response, closure); fut.wait(); + delete closure; return 0; } @@ -325,6 +329,7 @@ int HeterClient::Recv(const platform::DeviceContext& ctx, distributed::MultiVarMsg request; // 1. set req message_name(string) request.set_message_name(message_name); + request.set_group_id(0); // 2. set req recv_var_names() for (auto& recv_var_name : recv_var_names) { @@ -396,8 +401,8 @@ int HeterClient::Recv(int group_id, const std::vector& var_names, // save in worker auto& res_io_buffer = closure->cntl.response_attachment(); butil::IOBufBytesIterator io_buffer_itr(res_io_buffer); - io_buffer_itr.copy_and_forward(reinterpret_cast(data_ptr), - data_size * sizeof(float)); + io_buffer_itr.copy_and_forward(reinterpret_cast(data_ptr), data_size); + delete closure; VLOG(4) << "Recv done"; return 0; } diff --git a/paddle/fluid/distributed/ps/service/heter_client.h b/paddle/fluid/distributed/ps/service/heter_client.h old mode 100755 new mode 100644 index d1e0f21c7dd84..b9d65613399b2 --- a/paddle/fluid/distributed/ps/service/heter_client.h +++ b/paddle/fluid/distributed/ps/service/heter_client.h @@ -138,7 +138,8 @@ class HeterClient { const std::string& mode = "forward"); int Send(int group_id, const std::vector& var_names, - const std::vector& vars_len, void* data_ptr, int64_t data_size); + const std::vector& vars_len, void* data_ptr, + int64_t data_size); int Send(const platform::DeviceContext& ctx, const framework::Scope& scope, const std::string& message_name, diff --git a/paddle/fluid/distributed/ps/service/heter_server.cc b/paddle/fluid/distributed/ps/service/heter_server.cc index 292b12611c494..0753a6799c1be 100755 --- a/paddle/fluid/distributed/ps/service/heter_server.cc +++ b/paddle/fluid/distributed/ps/service/heter_server.cc @@ -20,8 +20,8 @@ namespace paddle { namespace distributed { // DEFINE_string(cert_path, "./cert.pem", "cert.pem path"); // DEFINE_string(key_path, "./key.pem", "key.pem path"); - std::shared_ptr HeterServer::s_instance_ = nullptr; +std::mutex HeterServer::mtx_; void HeterServer::RegisterServiceHandler(std::string message_name, HeterServiceHandler func) { @@ -130,21 +130,15 @@ int SendAndRecvVariableHandler::SaveInSwitchWithShard( butil::IOBufBytesIterator io_buffer_itr(request_io_buffer); for (int idx = 0; idx < request->send_var_names_size(); idx++) { const auto& var_name = request->send_var_names(idx); - const auto& var_len = request->vars_len(idx); - auto itr = local_shard.find(var_name); - if (itr != local_shard.end()) { - LOG(INFO) << "var: " << var_name << "has not been consumed!" - << "check again"; - WaitForVarsConsumed(group_id, var_name); - } + const auto& var_size = request->vars_len(idx); + WaitForVarsConsumed(group_id, var_name); auto& value = local_shard[var_name]; - value.resize(var_len); + value.resize(var_size); io_buffer_itr.copy_and_forward(reinterpret_cast(value.data()), - var_len * sizeof(float)); - VLOG(4) << "saved data in shards: "; - for (uint32_t i = 0; i < local_shard[var_name].size(); i++) { - VLOG(4) << *(local_shard[var_name].data() + i); - } + var_size); + std::unique_lock lk(scope_mutex_); + vars_ready_flag[group_id][var_name] = 1; + VLOG(4) << "saved var_name: " << var_name << "is saved ready!"; } VLOG(4) << "SaveInSwitchWithShard success"; return 0; @@ -164,20 +158,17 @@ int SendAndRecvVariableHandler::QueryInSwitchWithShard( } auto msg_name = request->message_name(); response->set_message_name(msg_name); - for (auto& req_var_name : req_var_names) { VLOG(4) << "req var name: " << req_var_name; response->add_send_var_names(req_var_name); + WaitForVarsProduced(group_id, req_var_name); auto itr = local_shard.find(req_var_name); - if (itr == local_shard.end()) { - LOG(INFO) << "var: " << req_var_name << " not found in shards"; - WaitForVarsProduced(group_id, req_var_name); - } - LOG(INFO) << "var: " << req_var_name << " found in shards"; - itr = local_shard.find(req_var_name); auto& value = itr.value(); - response_io_buffer.append(value.data(), value.size() * sizeof(float)); - value.resize(0); // 标记位 + response_io_buffer.append(value.data(), value.size()); + value.resize(0); // 清空内存 + std::unique_lock lk(scope_mutex_); + vars_ready_flag[group_id][req_var_name] = 0; + VLOG(4) << "query var_name: " << req_var_name << "is consumed ready!"; } VLOG(4) << "heter server QueryInSwitchWithShard done"; return 0; @@ -192,37 +183,31 @@ int SendAndRecvVariableHandler::SaveInSwitchWithScope( auto& cpu_dev_ctx = *pool.Get(cpu_place); auto message_name = request->message_name(); VLOG(4) << "message_name in heter server: " << message_name; + + auto send_var_nums = request->send_var_names_size(); + std::vector send_var_names(send_var_nums); + for (int idx = 0; idx < send_var_nums; idx++) { + send_var_names[idx] = request->var_messages(idx).varname(); + } std::unique_lock lk(scope_mutex_); auto local_scope = local_scope_ptr.get(); if (!local_scope) { LOG(ERROR) << "local_scope_ptr is null in SaveInSwitchWithScope"; } - for (int idx = 0; idx < request->send_var_names_size(); idx++) { - const auto& msg = request->var_messages(idx); - std::string var_name = msg.varname(); + for (auto var_name : send_var_names) { auto* var_exist_ptr = local_scope->FindVar(var_name); if (!var_exist_ptr) { VLOG(4) << "not find var: " << var_name << " in local_scope"; } - vars_table[var_name] += 1; - VLOG(4) << "saved var_name: " << var_name - << ", cnt = " << vars_table[var_name]; + WaitForVarsConsumed(0, var_name); } auto& request_io_buffer = cntl->request_attachment(); distributed::DeserializeFromMultiVarMsgAndIOBuf(*request, &request_io_buffer, cpu_dev_ctx, local_scope); lk.unlock(); - while (true) { - int ret = 0; - for (int idx = 0; idx < request->send_var_names_size(); idx++) { - ret |= vars_table[request->var_messages(idx).varname()]; - } - if (!ret) { - VLOG(4) << "all saved vars consumed"; - break; - } - VLOG(4) << "waiting consume result......"; - sleep(1); + for (auto var_name : send_var_names) { + std::unique_lock lk(scope_mutex_); + vars_ready_flag[0][var_name] = 1; } VLOG(4) << "SaveInSwitchWithScope success"; return 0; @@ -258,19 +243,14 @@ int SendAndRecvVariableHandler::QueryInSwitchWithScope( // 3. fill var_messages(VarMessage) for (auto& req_var_name : req_var_names) { - LOG(INFO) << "query var_name: " << req_var_name; + WaitForVarsProduced(0, req_var_name); auto* send_var_msg = response->add_var_messages(); send_var_msg->set_varname(req_var_name); framework::Variable* var_ptr; - while (true) { - var_ptr = local_scope->FindVar(req_var_name); - if (!var_ptr) { - LOG(INFO) << "local_scope not find var: " << req_var_name; - } else { - break; - } - sleep(1); + var_ptr = local_scope->FindVar(req_var_name); + if (!var_ptr) { + LOG(INFO) << "local_scope not find var: " << req_var_name; } butil::IOBuf temp_iobuf; if (var_ptr->IsType()) { @@ -282,10 +262,7 @@ int SendAndRecvVariableHandler::QueryInSwitchWithScope( } for (auto& req_var_name : req_var_names) { std::unique_lock lk(scope_mutex_); - vars_table[req_var_name] -= 1; - VLOG(4) << "remained var: " << req_var_name - << ", cnt = " << vars_table[req_var_name]; - lk.unlock(); + vars_ready_flag[0][req_var_name] = 0; } VLOG(4) << "heter server QueryInSwitchWithScope done"; return 0; diff --git a/paddle/fluid/distributed/ps/service/heter_server.h b/paddle/fluid/distributed/ps/service/heter_server.h index 624e76112c7b0..a65470cdbad5c 100644 --- a/paddle/fluid/distributed/ps/service/heter_server.h +++ b/paddle/fluid/distributed/ps/service/heter_server.h @@ -56,9 +56,10 @@ class Scope; DECLARE_double(eager_delete_tensor_gb); DECLARE_int32(pserver_timeout_ms); DECLARE_int32(heter_world_size); +DECLARE_int32(switch_send_recv_timeout_s); + namespace paddle { namespace distributed { - using MultiVarMsg = MultiVariableMessage; using VarMsg = VariableMessage; @@ -95,6 +96,19 @@ using SharedTaskQueue = std::shared_ptr< std::unordered_map>>>>; +class ValueInSwitch { + public: + ValueInSwitch() {} + ~ValueInSwitch() {} + char* data() { return _data.data(); } + size_t size() { return _data.size(); } + void resize(size_t size) { _data.resize(size); } + void shrink_to_fit() { _data.shrink_to_fit(); } + + private: + std::vector _data; +}; + class SendAndRecvVariableHandler final : public ServiceHandlerBase { public: SendAndRecvVariableHandler() { @@ -130,22 +144,31 @@ class SendAndRecvVariableHandler final : public ServiceHandlerBase { brpc::Controller* cntl); void WaitForVarsConsumed(int32_t group_id, const std::string& var_name) { - auto& local_shard = _local_shards[group_id]; - while (local_shard.find(var_name) != local_shard.end()) { - if (local_shard[var_name].size() == 0) { + timeline_.Start(); + while (true) { + if (vars_ready_flag[group_id][var_name] == 0) { + break; + } + timeline_.Pause(); + if (timeline_.ElapsedSec() > FLAGS_switch_send_recv_timeout_s) { + VLOG(0) << "vars not consumed exceed 10 miniutes"; break; } - VLOG(4) << "waiting consume result......"; - sleep(1); } return; } void WaitForVarsProduced(int32_t group_id, const std::string& var_name) { - auto& local_shard = _local_shards[group_id]; - while (local_shard.find(var_name) == local_shard.end()) { - VLOG(4) << "waiting produce result......"; - sleep(1); + timeline_.Start(); + while (true) { + if (vars_ready_flag[group_id][var_name] == 1) { + break; + } + timeline_.Pause(); + if (timeline_.ElapsedSec() > FLAGS_switch_send_recv_timeout_s) { + VLOG(0) << "vars not produced exceed 10 miniutes"; + break; + } } return; } @@ -245,10 +268,12 @@ class SendAndRecvVariableHandler final : public ServiceHandlerBase { } public: - using shard_type = SparseTableShard; + using shard_type = SparseTableShard; std::shared_ptr local_scope_ptr; // for switch - std::unordered_map vars_table; + std::unordered_map> + vars_ready_flag; std::unique_ptr _local_shards; + platform::Timer timeline_; private: // share with HeterPipelineTrainer @@ -576,8 +601,11 @@ class HeterServer { // HeterWrapper singleton static std::shared_ptr GetInstance() { - if (NULL == s_instance_) { - s_instance_.reset(new HeterServer()); + if (s_instance_ == nullptr) { + std::unique_lock lock(mtx_); + if (NULL == s_instance_) { + s_instance_.reset(new HeterServer()); + } } return s_instance_; } @@ -587,6 +615,7 @@ class HeterServer { private: static std::shared_ptr s_instance_; mutable std::mutex mutex_; + static std::mutex mtx_; std::condition_variable cv_; std::condition_variable condition_ready_; bool stoped_ = true; diff --git a/paddle/fluid/distributed/ps/service/ps_client.h b/paddle/fluid/distributed/ps/service/ps_client.h index 6f27b0eb04624..0d3d23be4e8d1 100644 --- a/paddle/fluid/distributed/ps/service/ps_client.h +++ b/paddle/fluid/distributed/ps/service/ps_client.h @@ -198,6 +198,7 @@ class PSClient { _msg_handler_map[msg_type] = handler; return 0; } + virtual int HandleClient2ClientMsg(int msg_type, int from_client_id, const std::string &msg) { auto itr = _msg_handler_map.find(msg_type); @@ -239,6 +240,46 @@ class PSClient { const float **update_values, size_t num) = 0; + // for save cache + virtual std::future CacheShuffle( + uint32_t table_id, const std::string &path, const std::string &mode, + const std::string &cache_threshold) { + VLOG(0) << "Did not implement"; + std::promise promise; + std::future fut = promise.get_future(); + promise.set_value(-1); + return fut; + } + + virtual std::future CacheShuffleMultiTable( + std::vector tables, const std::string &path, const std::string &mode, + const std::string &cache_threshold) { + VLOG(0) << "Did not implement"; + std::promise promise; + std::future fut = promise.get_future(); + promise.set_value(-1); + return fut; + } + + virtual std::future SaveCache(uint32_t table_id, + const std::string &path, + const std::string &mode) { + VLOG(0) << "Did not implement"; + std::promise promise; + std::future fut = promise.get_future(); + promise.set_value(-1); + return fut; + } + + virtual std::future GetCacheThreshold(uint32_t table_id, + double &cache_threshold) { + VLOG(0) << "Did not implement"; + std::promise promise; + std::future fut = promise.get_future(); + promise.set_value(-1); + return fut; + } + protected: virtual int32_t Initialize() = 0; size_t _client_id; diff --git a/paddle/fluid/distributed/ps/service/ps_service/graph_py_service.cc b/paddle/fluid/distributed/ps/service/ps_service/graph_py_service.cc index 92dfeb6818a28..ced51b8cbe383 100644 --- a/paddle/fluid/distributed/ps/service/ps_service/graph_py_service.cc +++ b/paddle/fluid/distributed/ps/service/ps_service/graph_py_service.cc @@ -35,35 +35,71 @@ std::vector GraphPyService::split(std::string& str, void GraphPyService::add_table_feat_conf(std::string table_name, std::string feat_name, std::string feat_dtype, - int32_t feat_shape) { - if (this->table_id_map.count(table_name)) { - this->table_feat_conf_table_name.push_back(table_name); - this->table_feat_conf_feat_name.push_back(feat_name); - this->table_feat_conf_feat_dtype.push_back(feat_dtype); - this->table_feat_conf_feat_shape.push_back(feat_shape); + int feat_shape) { + if (feature_to_id.find(table_name) != feature_to_id.end()) { + int idx = feature_to_id[table_name]; + VLOG(0) << "for table name" << table_name << " idx = " << idx; + if (table_feat_mapping[idx].find(feat_name) == + table_feat_mapping[idx].end()) { + VLOG(0) << "for table name not found,make a new one"; + int res = (int)table_feat_mapping[idx].size(); + table_feat_mapping[idx][feat_name] = res; + VLOG(0) << "seq id = " << table_feat_mapping[idx][feat_name]; + } + int feat_idx = table_feat_mapping[idx][feat_name]; + VLOG(0) << "table_name " << table_name << " mapping id " << idx; + VLOG(0) << " feat name " << feat_name << " feat id" << feat_idx; + if (feat_idx < table_feat_conf_feat_name[idx].size()) { + // overide + table_feat_conf_feat_name[idx][feat_idx] = feat_name; + table_feat_conf_feat_dtype[idx][feat_idx] = feat_dtype; + table_feat_conf_feat_shape[idx][feat_idx] = feat_shape; + } else { + // new + table_feat_conf_feat_name[idx].push_back(feat_name); + table_feat_conf_feat_dtype[idx].push_back(feat_dtype); + table_feat_conf_feat_shape[idx].push_back(feat_shape); + } } + VLOG(0) << "add conf over"; } -void add_graph_node(std::vector node_ids, +void add_graph_node(std::string name, std::vector node_ids, std::vector weight_list) {} -void remove_graph_node(std::vector node_ids) {} +void remove_graph_node(std::string name, std::vector node_ids) {} void GraphPyService::set_up(std::string ips_str, int shard_num, std::vector node_types, std::vector edge_types) { set_shard_num(shard_num); set_num_node_types(node_types.size()); - - for (size_t table_id = 0; table_id < node_types.size(); table_id++) { - this->table_id_map[node_types[table_id]] = this->table_id_map.size(); - } + /* + int num_node_types; + std::unordered_map edge_idx, feature_idx; + std::vector> table_feat_mapping; + std::vector> table_feat_conf_feat_name; + std::vector> table_feat_conf_feat_dtype; + std::vector> table_feat_conf_feat_shape; + */ + id_to_edge = edge_types; for (size_t table_id = 0; table_id < edge_types.size(); table_id++) { - this->table_id_map[edge_types[table_id]] = this->table_id_map.size(); + int res = (int)edge_to_id.size(); + edge_to_id[edge_types[table_id]] = res; + } + id_to_feature = node_types; + for (size_t table_id = 0; table_id < node_types.size(); table_id++) { + int res = (int)feature_to_id.size(); + feature_to_id[node_types[table_id]] = res; } + table_feat_mapping.resize(node_types.size()); + this->table_feat_conf_feat_name.resize(node_types.size()); + this->table_feat_conf_feat_dtype.resize(node_types.size()); + this->table_feat_conf_feat_shape.resize(node_types.size()); std::istringstream stream(ips_str); std::string ip; server_size = 0; std::vector ips_list = split(ips_str, ';'); int index = 0; + VLOG(0) << "start to build server"; for (auto ips : ips_list) { auto ip_and_port = split(ips, ':'); server_list.push_back(ip_and_port[0]); @@ -73,6 +109,7 @@ void GraphPyService::set_up(std::string ips_str, int shard_num, host_sign_list.push_back(ph_host.SerializeToString()); index++; } + VLOG(0) << "build server done"; } void GraphPyClient::start_client() { std::map> dense_regions; @@ -130,30 +167,29 @@ ::paddle::distributed::PSParameter GraphPyServer::GetServerProto() { server_service_proto->set_start_server_port(0); server_service_proto->set_server_thread_num(12); - for (auto& tuple : this->table_id_map) { - VLOG(0) << " make a new table " << tuple.second; - ::paddle::distributed::TableParameter* sparse_table_proto = - downpour_server_proto->add_downpour_table_param(); - std::vector feat_name; - std::vector feat_dtype; - std::vector feat_shape; - for (size_t i = 0; i < this->table_feat_conf_table_name.size(); i++) { - if (tuple.first == table_feat_conf_table_name[i]) { - feat_name.push_back(table_feat_conf_feat_name[i]); - feat_dtype.push_back(table_feat_conf_feat_dtype[i]); - feat_shape.push_back(table_feat_conf_feat_shape[i]); - } - } - std::string table_type; - if (tuple.second < this->num_node_types) { - table_type = "node"; - } else { - table_type = "edge"; - } + // for (auto& tuple : this->table_id_map) { + // VLOG(0) << " make a new table " << tuple.second; + ::paddle::distributed::TableParameter* sparse_table_proto = + downpour_server_proto->add_downpour_table_param(); + // std::vector feat_name; + // std::vector feat_dtype; + // std::vector feat_shape; + // for (size_t i = 0; i < this->table_feat_conf_table_name.size(); i++) { + // if (tuple.first == table_feat_conf_table_name[i]) { + // feat_name.push_back(table_feat_conf_feat_name[i]); + // feat_dtype.push_back(table_feat_conf_feat_dtype[i]); + // feat_shape.push_back(table_feat_conf_feat_shape[i]); + // } + // } + // std::string table_type; + // if (tuple.second < this->num_node_types) { + // table_type = "node"; + // } else { + // table_type = "edge"; + // } - GetDownpourSparseTableProto(sparse_table_proto, tuple.second, tuple.first, - table_type, feat_name, feat_dtype, feat_shape); - } + GetDownpourSparseTableProto(sparse_table_proto); + //} return server_fleet_desc; } @@ -166,31 +202,29 @@ ::paddle::distributed::PSParameter GraphPyClient::GetWorkerProto() { ::paddle::distributed::DownpourWorkerParameter* downpour_worker_proto = worker_proto->mutable_downpour_worker_param(); - for (auto& tuple : this->table_id_map) { - VLOG(0) << " make a new table " << tuple.second; - ::paddle::distributed::TableParameter* worker_sparse_table_proto = - downpour_worker_proto->add_downpour_table_param(); - std::vector feat_name; - std::vector feat_dtype; - std::vector feat_shape; - for (size_t i = 0; i < this->table_feat_conf_table_name.size(); i++) { - if (tuple.first == table_feat_conf_table_name[i]) { - feat_name.push_back(table_feat_conf_feat_name[i]); - feat_dtype.push_back(table_feat_conf_feat_dtype[i]); - feat_shape.push_back(table_feat_conf_feat_shape[i]); - } - } - std::string table_type; - if (tuple.second < this->num_node_types) { - table_type = "node"; - } else { - table_type = "edge"; - } + // for (auto& tuple : this->table_id_map) { + // VLOG(0) << " make a new table " << tuple.second; + ::paddle::distributed::TableParameter* worker_sparse_table_proto = + downpour_worker_proto->add_downpour_table_param(); + // std::vector feat_name; + // std::vector feat_dtype; + // std::vector feat_shape; + // for (size_t i = 0; i < this->table_feat_conf_table_name.size(); i++) { + // if (tuple.first == table_feat_conf_table_name[i]) { + // feat_name.push_back(table_feat_conf_feat_name[i]); + // feat_dtype.push_back(table_feat_conf_feat_dtype[i]); + // feat_shape.push_back(table_feat_conf_feat_shape[i]); + // } + // } + // std::string table_type; + // if (tuple.second < this->num_node_types) { + // table_type = "node"; + // } else { + // table_type = "edge"; + // } - GetDownpourSparseTableProto(worker_sparse_table_proto, tuple.second, - tuple.first, table_type, feat_name, feat_dtype, - feat_shape); - } + GetDownpourSparseTableProto(worker_sparse_table_proto); + //} ::paddle::distributed::ServerParameter* server_proto = worker_fleet_desc.mutable_server_param(); @@ -204,30 +238,29 @@ ::paddle::distributed::PSParameter GraphPyClient::GetWorkerProto() { server_service_proto->set_start_server_port(0); server_service_proto->set_server_thread_num(12); - for (auto& tuple : this->table_id_map) { - VLOG(0) << " make a new table " << tuple.second; - ::paddle::distributed::TableParameter* sparse_table_proto = - downpour_server_proto->add_downpour_table_param(); - std::vector feat_name; - std::vector feat_dtype; - std::vector feat_shape; - for (size_t i = 0; i < this->table_feat_conf_table_name.size(); i++) { - if (tuple.first == table_feat_conf_table_name[i]) { - feat_name.push_back(table_feat_conf_feat_name[i]); - feat_dtype.push_back(table_feat_conf_feat_dtype[i]); - feat_shape.push_back(table_feat_conf_feat_shape[i]); - } - } - std::string table_type; - if (tuple.second < this->num_node_types) { - table_type = "node"; - } else { - table_type = "edge"; - } + // for (auto& tuple : this->table_id_map) { + // VLOG(0) << " make a new table " << tuple.second; + ::paddle::distributed::TableParameter* sparse_table_proto = + downpour_server_proto->add_downpour_table_param(); + // std::vector feat_name; + // std::vector feat_dtype; + // std::vector feat_shape; + // for (size_t i = 0; i < this->table_feat_conf_table_name.size(); i++) { + // if (tuple.first == table_feat_conf_table_name[i]) { + // feat_name.push_back(table_feat_conf_feat_name[i]); + // feat_dtype.push_back(table_feat_conf_feat_dtype[i]); + // feat_shape.push_back(table_feat_conf_feat_shape[i]); + // } + // } + // std::string table_type; + // if (tuple.second < this->num_node_types) { + // table_type = "node"; + // } else { + // table_type = "edge"; + // } - GetDownpourSparseTableProto(sparse_table_proto, tuple.second, tuple.first, - table_type, feat_name, feat_dtype, feat_shape); - } + GetDownpourSparseTableProto(sparse_table_proto); + //} return worker_fleet_desc; } @@ -237,57 +270,88 @@ void GraphPyClient::load_edge_file(std::string name, std::string filepath, std::string params = "e"; if (reverse) { // 'e<' means load edges from $2 to $1 - params += "<"; + params += "<" + name; } else { // 'e>' means load edges from $1 to $2 - params += ">"; + params += ">" + name; } - if (this->table_id_map.count(name)) { - VLOG(0) << "loadding data with type " << name << " from " << filepath; - uint32_t table_id = this->table_id_map[name]; - auto status = - get_ps_client()->Load(table_id, std::string(filepath), params); + if (edge_to_id.find(name) != edge_to_id.end()) { + auto status = get_ps_client()->Load(0, std::string(filepath), params); status.wait(); } + // if (this->table_id_map.count(name)) { + // VLOG(0) << "loadding data with type " << name << " from " << filepath; + // uint32_t table_id = this->table_id_map[name]; + // auto status = + // get_ps_client()->Load(table_id, std::string(filepath), params); + // status.wait(); + // } } void GraphPyClient::clear_nodes(std::string name) { - if (this->table_id_map.count(name)) { - uint32_t table_id = this->table_id_map[name]; - auto status = get_ps_client()->clear_nodes(table_id); + if (edge_to_id.find(name) != edge_to_id.end()) { + int idx = edge_to_id[name]; + auto status = get_ps_client()->clear_nodes(0, 0, idx); + status.wait(); + } else if (feature_to_id.find(name) != feature_to_id.end()) { + int idx = feature_to_id[name]; + auto status = get_ps_client()->clear_nodes(0, 1, idx); status.wait(); } + + // if (this->table_id_map.count(name)) { + // uint32_t table_id = this->table_id_map[name]; + // auto status = get_ps_client()->clear_nodes(table_id); + // status.wait(); + // } } void GraphPyClient::add_graph_node(std::string name, std::vector& node_ids, std::vector& weight_list) { - if (this->table_id_map.count(name)) { - uint32_t table_id = this->table_id_map[name]; + // if (this->table_id_map.count(name)) { + // uint32_t table_id = this->table_id_map[name]; + // auto status = + // get_ps_client()->add_graph_node(table_id, node_ids, weight_list); + // status.wait(); + // } + if (edge_to_id.find(name) != edge_to_id.end()) { + int idx = edge_to_id[name]; auto status = - get_ps_client()->add_graph_node(table_id, node_ids, weight_list); + get_ps_client()->add_graph_node(0, idx, node_ids, weight_list); status.wait(); } } void GraphPyClient::remove_graph_node(std::string name, std::vector& node_ids) { - if (this->table_id_map.count(name)) { - uint32_t table_id = this->table_id_map[name]; - auto status = get_ps_client()->remove_graph_node(table_id, node_ids); + if (edge_to_id.find(name) != edge_to_id.end()) { + int idx = edge_to_id[name]; + auto status = get_ps_client()->remove_graph_node(0, idx, node_ids); status.wait(); } + // if (this->table_id_map.count(name)) { + // uint32_t table_id = this->table_id_map[name]; + // auto status = get_ps_client()->remove_graph_node(table_id, node_ids); + // status.wait(); + // } } void GraphPyClient::load_node_file(std::string name, std::string filepath) { // 'n' means load nodes and 'node_type' follows + std::string params = "n" + name; - if (this->table_id_map.count(name)) { - uint32_t table_id = this->table_id_map[name]; - auto status = - get_ps_client()->Load(table_id, std::string(filepath), params); + + if (feature_to_id.find(name) != feature_to_id.end()) { + auto status = get_ps_client()->Load(0, std::string(filepath), params); status.wait(); } + // if (this->table_id_map.count(name)) { + // uint32_t table_id = this->table_id_map[name]; + // auto status = + // get_ps_client()->Load(table_id, std::string(filepath), params); + // status.wait(); + // } } std::pair>, std::vector> @@ -297,12 +361,18 @@ GraphPyClient::batch_sample_neighbors(std::string name, bool return_edges) { std::vector> v; std::vector> v1; - if (this->table_id_map.count(name)) { - uint32_t table_id = this->table_id_map[name]; - auto status = worker_ptr->batch_sample_neighbors( - table_id, node_ids, sample_size, v, v1, return_weight); + if (edge_to_id.find(name) != edge_to_id.end()) { + int idx = edge_to_id[name]; + auto status = get_ps_client()->batch_sample_neighbors( + 0, idx, node_ids, sample_size, v, v1, return_weight); status.wait(); } + // if (this->table_id_map.count(name)) { + // uint32_t table_id = this->table_id_map[name]; + // auto status = worker_ptr->batch_sample_neighbors( + // table_id, node_ids, sample_size, v, v1, return_weight); + // status.wait(); + // } // res.first[0]: neighbors (nodes) // res.first[1]: slice index @@ -331,54 +401,70 @@ GraphPyClient::batch_sample_neighbors(std::string name, return res; } -void GraphPyClient::use_neighbors_sample_cache(std::string name, - size_t total_size_limit, - size_t ttl) { - if (this->table_id_map.count(name)) { - uint32_t table_id = this->table_id_map[name]; - auto status = - worker_ptr->use_neighbors_sample_cache(table_id, total_size_limit, ttl); - status.wait(); - } -} std::vector GraphPyClient::random_sample_nodes(std::string name, int server_index, int sample_size) { std::vector v; - if (this->table_id_map.count(name)) { - uint32_t table_id = this->table_id_map[name]; - auto status = - worker_ptr->random_sample_nodes(table_id, server_index, sample_size, v); + if (feature_to_id.find(name) != feature_to_id.end()) { + int idx = feature_to_id[name]; + auto status = get_ps_client()->random_sample_nodes(0, 1, idx, server_index, + sample_size, v); + status.wait(); + } else if (edge_to_id.find(name) != edge_to_id.end()) { + int idx = edge_to_id[name]; + auto status = get_ps_client()->random_sample_nodes(0, 0, idx, server_index, + sample_size, v); status.wait(); } + // if (this->table_id_map.count(name)) { + // uint32_t table_id = this->table_id_map[name]; + // auto status = + // worker_ptr->random_sample_nodes(table_id, server_index, sample_size, + // v); + // status.wait(); + // } return v; } // (name, dtype, ndarray) std::vector> GraphPyClient::get_node_feat( - std::string node_type, std::vector node_ids, + std::string name, std::vector node_ids, std::vector feature_names) { std::vector> v( feature_names.size(), std::vector(node_ids.size())); - if (this->table_id_map.count(node_type)) { - uint32_t table_id = this->table_id_map[node_type]; + if (feature_to_id.find(name) != feature_to_id.end()) { + int idx = feature_to_id[name]; auto status = - worker_ptr->get_node_feat(table_id, node_ids, feature_names, v); + get_ps_client()->get_node_feat(0, idx, node_ids, feature_names, v); status.wait(); } + // if (this->table_id_map.count(node_type)) { + // uint32_t table_id = this->table_id_map[node_type]; + // auto status = + // worker_ptr->get_node_feat(table_id, node_ids, feature_names, v); + // status.wait(); + // } return v; } void GraphPyClient::set_node_feat( - std::string node_type, std::vector node_ids, + std::string name, std::vector node_ids, std::vector feature_names, const std::vector> features) { - if (this->table_id_map.count(node_type)) { - uint32_t table_id = this->table_id_map[node_type]; - auto status = - worker_ptr->set_node_feat(table_id, node_ids, feature_names, features); + if (feature_to_id.find(name) != feature_to_id.end()) { + int idx = feature_to_id[name]; + auto status = get_ps_client()->set_node_feat(0, idx, node_ids, + feature_names, features); status.wait(); } + + // if (this->table_id_map.count(node_type)) { + // uint32_t table_id = this->table_id_map[node_type]; + // auto status = + // worker_ptr->set_node_feat(table_id, node_ids, feature_names, + // features); + // status.wait(); + // } return; } @@ -387,10 +473,21 @@ std::vector GraphPyClient::pull_graph_list(std::string name, int start, int size, int step) { std::vector res; - if (this->table_id_map.count(name)) { - uint32_t table_id = this->table_id_map[name]; - auto status = worker_ptr->pull_graph_list(table_id, server_index, start, - size, step, res); + // if (this->table_id_map.count(name)) { + // uint32_t table_id = this->table_id_map[name]; + // auto status = worker_ptr->pull_graph_list(table_id, server_index, start, + // size, step, res); + // status.wait(); + // } + if (feature_to_id.find(name) != feature_to_id.end()) { + int idx = feature_to_id[name]; + auto status = get_ps_client()->pull_graph_list(0, 1, idx, server_index, + start, size, step, res); + status.wait(); + } else if (edge_to_id.find(name) != edge_to_id.end()) { + int idx = edge_to_id[name]; + auto status = get_ps_client()->pull_graph_list(0, 0, idx, server_index, + start, size, step, res); status.wait(); } return res; diff --git a/paddle/fluid/distributed/ps/service/ps_service/graph_py_service.h b/paddle/fluid/distributed/ps/service/ps_service/graph_py_service.h index 19f34dad80745..55beb9b3932a6 100644 --- a/paddle/fluid/distributed/ps/service/ps_service/graph_py_service.h +++ b/paddle/fluid/distributed/ps/service/ps_service/graph_py_service.h @@ -49,21 +49,19 @@ class GraphPyService { std::vector server_list, port_list, host_sign_list; int server_size, shard_num; int num_node_types; - std::unordered_map table_id_map; - std::vector table_feat_conf_table_name; - std::vector table_feat_conf_feat_name; - std::vector table_feat_conf_feat_dtype; - std::vector table_feat_conf_feat_shape; + std::unordered_map edge_to_id, feature_to_id; + std::vector id_to_feature, id_to_edge; + std::vector> table_feat_mapping; + std::vector> table_feat_conf_feat_name; + std::vector> table_feat_conf_feat_dtype; + std::vector> table_feat_conf_feat_shape; public: int get_shard_num() { return shard_num; } void set_shard_num(int shard_num) { this->shard_num = shard_num; } void GetDownpourSparseTableProto( - ::paddle::distributed::TableParameter* sparse_table_proto, - uint32_t table_id, std::string table_name, std::string table_type, - std::vector feat_name, std::vector feat_dtype, - std::vector feat_shape) { - sparse_table_proto->set_table_id(table_id); + ::paddle::distributed::TableParameter* sparse_table_proto) { + sparse_table_proto->set_table_id(0); sparse_table_proto->set_table_class("GraphTable"); sparse_table_proto->set_shard_num(shard_num); sparse_table_proto->set_type(::paddle::distributed::PS_SPARSE_TABLE); @@ -76,14 +74,26 @@ class GraphPyService { ::paddle::distributed::GraphParameter* graph_proto = sparse_table_proto->mutable_graph_parameter(); - ::paddle::distributed::GraphFeature* graph_feature = - graph_proto->mutable_graph_feature(); + // ::paddle::distributed::GraphFeature* graph_feature = + // graph_proto->mutable_graph_feature(); graph_proto->set_task_pool_size(24); - graph_proto->set_table_name(table_name); - graph_proto->set_table_type(table_type); + graph_proto->set_table_name("cpu_graph_table"); graph_proto->set_use_cache(false); + for (int i = 0; i < id_to_edge.size(); i++) + graph_proto->add_edge_types(id_to_edge[i]); + for (int i = 0; i < id_to_feature.size(); i++) { + graph_proto->add_node_types(id_to_feature[i]); + auto feat_node = id_to_feature[i]; + ::paddle::distributed::GraphFeature* g_f = + graph_proto->add_graph_feature(); + for (int x = 0; x < table_feat_conf_feat_name[i].size(); x++) { + g_f->add_name(table_feat_conf_feat_name[i][x]); + g_f->add_dtype(table_feat_conf_feat_dtype[i][x]); + g_f->add_shape(table_feat_conf_feat_shape[i][x]); + } + } // Set GraphTable Parameter // common_proto->set_table_name(table_name); // common_proto->set_name(table_type); @@ -93,11 +103,11 @@ class GraphPyService { // common_proto->add_attributes(feat_name[i]); // } - for (size_t i = 0; i < feat_name.size(); i++) { - graph_feature->add_dtype(feat_dtype[i]); - graph_feature->add_shape(feat_shape[i]); - graph_feature->add_name(feat_name[i]); - } + // for (size_t i = 0; i < feat_name.size(); i++) { + // graph_feature->add_dtype(feat_dtype[i]); + // graph_feature->add_shape(feat_shape[i]); + // graph_feature->add_name(feat_name[i]); + // } accessor_proto->set_accessor_class("CommMergeAccessor"); } @@ -172,10 +182,8 @@ class GraphPyClient : public GraphPyService { std::vector random_sample_nodes(std::string name, int server_index, int sample_size); std::vector> get_node_feat( - std::string node_type, std::vector node_ids, + std::string name, std::vector node_ids, std::vector feature_names); - void use_neighbors_sample_cache(std::string name, size_t total_size_limit, - size_t ttl); void set_node_feat(std::string node_type, std::vector node_ids, std::vector feature_names, const std::vector> features); diff --git a/paddle/fluid/distributed/ps/service/sendrecv.proto b/paddle/fluid/distributed/ps/service/sendrecv.proto index 580f411c28c07..ae6364dd8371e 100755 --- a/paddle/fluid/distributed/ps/service/sendrecv.proto +++ b/paddle/fluid/distributed/ps/service/sendrecv.proto @@ -65,6 +65,8 @@ enum PsCmdID { PS_SAVE_WITH_SHARD = 44; PS_QUERY_WITH_SCOPE = 45; PS_QUERY_WITH_SHARD = 46; + // pserver2pserver cmd start from 100 + PS_S2S_MSG = 101; } message PsRequestMessage { @@ -124,7 +126,7 @@ message MultiVariableMessage { repeated string recv_var_names = 3; repeated VariableMessage var_messages = 4; optional bytes data = 5; - repeated int32 vars_len = 6; + repeated int64 vars_len = 6; optional int32 group_id = 7; }; diff --git a/paddle/fluid/distributed/ps/service/server.cc b/paddle/fluid/distributed/ps/service/server.cc index 65f7ae821cef1..a6e0f39474b06 100644 --- a/paddle/fluid/distributed/ps/service/server.cc +++ b/paddle/fluid/distributed/ps/service/server.cc @@ -67,6 +67,8 @@ int32_t PSServer::Configure( _config = config.server_param(); _rank = server_rank; _environment = &env; + _shuffled_ins = + paddle::framework::MakeChannel>(); size_t shard_num = env.GetPsServers().size(); const auto &downpour_param = _config.downpour_server_param(); diff --git a/paddle/fluid/distributed/ps/service/server.h b/paddle/fluid/distributed/ps/service/server.h index 5da819326b052..c044e82884604 100644 --- a/paddle/fluid/distributed/ps/service/server.h +++ b/paddle/fluid/distributed/ps/service/server.h @@ -89,6 +89,45 @@ class PSServer { return &_table_map; } + // for cache + virtual int32_t StartS2S() { return 0; } + + virtual ::std::future SendPServer2PServerMsg( + int msg_type, int to_pserver_id, const std::string &msg) { + LOG(FATAL) << "NotImplementError: PSServer::send_pserver2pserver_msg"; + std::promise promise; + std::future fut = promise.get_future(); + promise.set_value(-1); + return fut; + } + + typedef std::function MsgHandlerFunc; + virtual int RegistePServer2PServerMsgHandler(int msg_type, + MsgHandlerFunc handler) { + _msg_handler_map[msg_type] = handler; + return 0; + } + virtual int HandlePServer2PServerMsg(int msg_type, int from_pserver_id, + const std::string &msg) { + auto itr = _msg_handler_map.find(msg_type); + if (itr == _msg_handler_map.end()) { + if (msg_type == 101) { + return ReceiveFromPServer(msg_type, from_pserver_id, msg); + } else { + LOG(WARNING) << "unknown pserver2pserver_msg type:" << msg_type; + return -1; + } + } + return itr->second(msg_type, from_pserver_id, msg); + } + virtual int32_t ReceiveFromPServer(int msg_type, int pserver_id, + const std::string &msg) { + LOG(FATAL) << "NotImplementError::PSServer::ReceiveFromPServer"; + return -1; + } + + paddle::framework::Channel> _shuffled_ins; + protected: virtual int32_t Initialize() = 0; @@ -97,6 +136,7 @@ class PSServer { ServerParameter _config; PSEnvironment *_environment; std::unordered_map> _table_map; + std::unordered_map _msg_handler_map; protected: std::shared_ptr scope_; diff --git a/paddle/fluid/distributed/ps/table/CMakeLists.txt b/paddle/fluid/distributed/ps/table/CMakeLists.txt index bb6725b08425a..f2b9eb71f5a64 100644 --- a/paddle/fluid/distributed/ps/table/CMakeLists.txt +++ b/paddle/fluid/distributed/ps/table/CMakeLists.txt @@ -18,17 +18,12 @@ include_directories(${PADDLE_LIB_THIRD_PARTY_PATH}libmct/src/extern_libmct/libmc set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fopenmp") -set(EXTERN_DEP "") -if(WITH_HETERPS) - set(TABLE_SRC memory_dense_table.cc barrier_table.cc common_graph_table.cc) - set(EXTERN_DEP rocksdb) -else() - set(TABLE_SRC memory_dense_table.cc barrier_table.cc common_graph_table.cc) -endif() +set(TABLE_SRC memory_dense_table.cc barrier_table.cc common_graph_table.cc) +#set(EXTERN_DEP rocksdb) cc_library(common_table SRCS ${TABLE_SRC} DEPS ${TABLE_DEPS} ${RPC_DEPS} graph_edge graph_node device_context string_helper -simple_threadpool xxhash generator ${EXTERN_DEP}) +simple_threadpool xxhash generator) set_source_files_properties(tensor_accessor.cc PROPERTIES COMPILE_FLAGS ${DISTRIBUTE_COMPILE_FLAGS}) @@ -41,13 +36,13 @@ set_source_files_properties(ctr_double_accessor.cc PROPERTIES COMPILE_FLAGS ${DI set_source_files_properties(ctr_accessor.cc PROPERTIES COMPILE_FLAGS ${DISTRIBUTE_COMPILE_FLAGS}) set_source_files_properties(sparse_accessor.cc PROPERTIES COMPILE_FLAGS ${DISTRIBUTE_COMPILE_FLAGS}) set_source_files_properties(memory_sparse_table.cc PROPERTIES COMPILE_FLAGS ${DISTRIBUTE_COMPILE_FLAGS}) +set_source_files_properties(ssd_sparse_table.cc PROPERTIES COMPILE_FLAGS ${DISTRIBUTE_COMPILE_FLAGS}) +set_source_files_properties(memory_sparse_geo_table.cc PROPERTIES COMPILE_FLAGS ${DISTRIBUTE_COMPILE_FLAGS}) + cc_library(sparse_sgd_rule SRCS sparse_sgd_rule.cc DEPS ${TABLE_DEPS} ps_framework_proto) cc_library(ctr_accessor SRCS ctr_accessor.cc ctr_double_accessor.cc sparse_accessor.cc DEPS ${TABLE_DEPS} ps_framework_proto sparse_sgd_rule) -cc_library(memory_sparse_table SRCS memory_sparse_table.cc DEPS ps_framework_proto ${TABLE_DEPS} fs afs_wrapper ctr_accessor common_table) - -set_source_files_properties(memory_sparse_geo_table.cc PROPERTIES COMPILE_FLAGS ${DISTRIBUTE_COMPILE_FLAGS}) -cc_library(memory_sparse_geo_table SRCS memory_sparse_geo_table.cc DEPS ps_framework_proto ${TABLE_DEPS} common_table) +cc_library(sparse_table SRCS memory_sparse_table.cc ssd_sparse_table.cc memory_sparse_geo_table.cc DEPS ps_framework_proto ${TABLE_DEPS} fs afs_wrapper ctr_accessor common_table rocksdb) -cc_library(table SRCS table.cc DEPS memory_sparse_table memory_sparse_geo_table common_table tensor_accessor tensor_table ps_framework_proto string_helper device_context gflags glog boost) +cc_library(table SRCS table.cc DEPS sparse_table common_table tensor_accessor tensor_table ps_framework_proto string_helper device_context gflags glog boost) target_link_libraries(table -fopenmp) diff --git a/paddle/fluid/distributed/ps/table/accessor.h b/paddle/fluid/distributed/ps/table/accessor.h index 024af327a33af..7713c2bda295f 100644 --- a/paddle/fluid/distributed/ps/table/accessor.h +++ b/paddle/fluid/distributed/ps/table/accessor.h @@ -117,6 +117,11 @@ class ValueAccessor { virtual bool Save(float* value, int param) = 0; // update delta_score and unseen_days after save virtual void UpdateStatAfterSave(float* value, int param) {} + // 判断该value是否保存到ssd + virtual bool SaveSSD(float* value) = 0; + // + virtual bool SaveCache(float* value, int param, + double global_cache_threshold) = 0; // keys不存在时，为values生成随机值 virtual int32_t Create(float** value, size_t num) = 0; diff --git a/paddle/fluid/distributed/ps/table/common_graph_table.cc b/paddle/fluid/distributed/ps/table/common_graph_table.cc index d7ceb4a18ea19..a9cd0021c8578 100644 --- a/paddle/fluid/distributed/ps/table/common_graph_table.cc +++ b/paddle/fluid/distributed/ps/table/common_graph_table.cc @@ -29,7 +29,7 @@ namespace distributed { #ifdef PADDLE_WITH_HETERPS paddle::framework::GpuPsCommGraph GraphTable::make_gpu_ps_graph( - std::vector ids) { + int idx, std::vector ids) { std::vector> bags(task_pool_size_); for (auto x : ids) { int location = x % shard_num % task_pool_size_; @@ -43,7 +43,7 @@ paddle::framework::GpuPsCommGraph GraphTable::make_gpu_ps_graph( tasks.push_back(_shards_task_pool[i]->enqueue([&, i, this]() -> int { paddle::framework::GpuPsGraphNode x; for (int j = 0; j < (int)bags[i].size(); j++) { - Node *v = find_node(bags[i][j]); + Node *v = find_node(0, idx, bags[i][j]); x.node_id = bags[i][j]; if (v == NULL) { x.neighbor_size = 0; @@ -85,22 +85,32 @@ paddle::framework::GpuPsCommGraph GraphTable::make_gpu_ps_graph( } return res; } -int32_t GraphTable::add_node_to_ssd(int64_t src_id, char *data, int len) { - if (_db != NULL) - _db->put(src_id % shard_num % task_pool_size_, (char *)&src_id, - sizeof(uint64_t), (char *)data, sizeof(int64_t) * len); +int32_t GraphTable::add_node_to_ssd(int type_id, int idx, int64_t src_id, + char *data, int len) { + if (_db != NULL) { + char ch[sizeof(int) * 2 + sizeof(int64_t)]; + memcpy(ch, &type_id, sizeof(int)); + memcpy(ch + sizeof(int), &idx, sizeof(int)); + memcpy(ch + sizeof(int) * 2, &src_id, sizeof(int64_t)); + _db->put(src_id % shard_num % task_pool_size_, ch, + sizeof(int) * 2 + sizeof(int64_t), (char *)data, len); + } return 0; } char *GraphTable::random_sample_neighbor_from_ssd( - int64_t id, int sample_size, const std::shared_ptr rng, - int &actual_size) { + int idx, int64_t id, int sample_size, + const std::shared_ptr rng, int &actual_size) { if (_db == NULL) { actual_size = 0; return NULL; } std::string str; - if (_db->get(id % shard_num % task_pool_size_, (char *)&id, sizeof(uint64_t), - str) == 0) { + char ch[sizeof(int) * 2 + sizeof(int64_t)]; + memset(ch, 0, sizeof(int)); + memcpy(ch + sizeof(int), &idx, sizeof(int)); + memcpy(ch + sizeof(int) * 2, &id, sizeof(int64_t)); + if (_db->get(id % shard_num % task_pool_size_, ch, sizeof(uint64_t), str) == + 0) { int64_t *data = ((int64_t *)str.c_str()); int n = str.size() / sizeof(int64_t); std::unordered_map m; @@ -423,20 +433,20 @@ std::vector GraphShard::get_batch(int start, int end, int step) { size_t GraphShard::get_size() { return bucket.size(); } -int32_t GraphTable::add_comm_edge(int64_t src_id, int64_t dst_id) { +int32_t GraphTable::add_comm_edge(int idx, int64_t src_id, int64_t dst_id) { size_t src_shard_id = src_id % shard_num; if (src_shard_id >= shard_end || src_shard_id < shard_start) { return -1; } size_t index = src_shard_id - shard_start; - VLOG(0) << "index add edge " << src_id << " " << dst_id; - shards[index]->add_graph_node(src_id)->build_edges(false); - shards[index]->add_neighbor(src_id, dst_id, 1.0); + edge_shards[idx][index]->add_graph_node(src_id)->build_edges(false); + edge_shards[idx][index]->add_neighbor(src_id, dst_id, 1.0); return 0; } -int32_t GraphTable::add_graph_node(std::vector &id_list, +int32_t GraphTable::add_graph_node(int idx, std::vector &id_list, std::vector &is_weight_list) { + auto &shards = edge_shards[idx]; size_t node_size = id_list.size(); std::vector>> batch(task_pool_size_); for (size_t i = 0; i < node_size; i++) { @@ -450,19 +460,20 @@ int32_t GraphTable::add_graph_node(std::vector &id_list, std::vector> tasks; for (size_t i = 0; i < batch.size(); ++i) { if (!batch[i].size()) continue; - tasks.push_back(_shards_task_pool[i]->enqueue([&batch, i, this]() -> int { - for (auto &p : batch[i]) { - size_t index = p.first % this->shard_num - this->shard_start; - this->shards[index]->add_graph_node(p.first)->build_edges(p.second); - } - return 0; - })); + tasks.push_back( + _shards_task_pool[i]->enqueue([&shards, &batch, i, this]() -> int { + for (auto &p : batch[i]) { + size_t index = p.first % this->shard_num - this->shard_start; + shards[index]->add_graph_node(p.first)->build_edges(p.second); + } + return 0; + })); } for (size_t i = 0; i < tasks.size(); i++) tasks[i].get(); return 0; } -int32_t GraphTable::remove_graph_node(std::vector &id_list) { +int32_t GraphTable::remove_graph_node(int idx, std::vector &id_list) { size_t node_size = id_list.size(); std::vector> batch(task_pool_size_); for (size_t i = 0; i < node_size; i++) { @@ -470,16 +481,18 @@ int32_t GraphTable::remove_graph_node(std::vector &id_list) { if (shard_id >= shard_end || shard_id < shard_start) continue; batch[get_thread_pool_index(id_list[i])].push_back(id_list[i]); } + auto &shards = edge_shards[idx]; std::vector> tasks; for (size_t i = 0; i < batch.size(); ++i) { if (!batch[i].size()) continue; - tasks.push_back(_shards_task_pool[i]->enqueue([&batch, i, this]() -> int { - for (auto &p : batch[i]) { - size_t index = p % this->shard_num - this->shard_start; - this->shards[index]->delete_node(p); - } - return 0; - })); + tasks.push_back( + _shards_task_pool[i]->enqueue([&shards, &batch, i, this]() -> int { + for (auto &p : batch[i]) { + size_t index = p % this->shard_num - this->shard_start; + shards[index]->delete_node(p); + } + return 0; + })); } for (size_t i = 0; i < tasks.size(); i++) tasks[i].get(); return 0; @@ -541,30 +554,19 @@ Node *GraphShard::find_node(int64_t id) { } GraphTable::~GraphTable() { - for (auto p : shards) { - delete p; - } - for (auto p : extra_shards) { - delete p; + for (int i = 0; i < (int)edge_shards.size(); i++) { + for (auto p : edge_shards[i]) { + delete p; + } + edge_shards[i].clear(); } - shards.clear(); - extra_shards.clear(); -} -int32_t GraphTable::load_graph_split_config(const std::string &path) { - VLOG(4) << "in server side load graph split config\n"; - std::ifstream file(path); - std::string line; - while (std::getline(file, line)) { - auto values = paddle::string::split_string(line, "\t"); - if (values.size() < 2) continue; - size_t index = (size_t)std::stoi(values[0]); - if (index != _shard_idx) continue; - auto dst_id = std::stoull(values[1]); - extra_nodes.insert(dst_id); - } - if (extra_nodes.size() != 0) use_duplicate_nodes = true; - return 0; + for (int i = 0; i < (int)feature_shards.size(); i++) { + for (auto p : feature_shards[i]) { + delete p; + } + feature_shards[i].clear(); + } } int32_t GraphTable::Load(const std::string &path, const std::string ¶m) { @@ -572,7 +574,8 @@ int32_t GraphTable::Load(const std::string &path, const std::string ¶m) { bool load_node = (param[0] == 'n'); if (load_edge) { bool reverse_edge = (param[1] == '<'); - return this->load_edges(path, reverse_edge); + std::string edge_type = param.substr(2); + return this->load_edges(path, reverse_edge, edge_type); } if (load_node) { std::string node_type = param.substr(1); @@ -582,9 +585,11 @@ int32_t GraphTable::Load(const std::string &path, const std::string ¶m) { } int32_t GraphTable::get_nodes_ids_by_ranges( - std::vector> ranges, std::vector &res) { + int type_id, int idx, std::vector> ranges, + std::vector &res) { int start = 0, end, index = 0, total_size = 0; res.clear(); + auto &shards = type_id == 0 ? edge_shards[idx] : feature_shards[idx]; std::vector>> tasks; for (size_t i = 0; i < shards.size() && index < (int)ranges.size(); i++) { end = total_size + shards[i]->get_size(); @@ -601,7 +606,7 @@ int32_t GraphTable::get_nodes_ids_by_ranges( first -= total_size; second -= total_size; tasks.push_back(_shards_task_pool[i % task_pool_size_]->enqueue( - [this, first, second, i]() -> std::vector { + [&shards, this, first, second, i]() -> std::vector { return shards[i]->get_ids_by_range(first, second); })); } @@ -622,6 +627,18 @@ int32_t GraphTable::load_nodes(const std::string &path, std::string node_type) { auto paths = paddle::string::split_string(path, ";"); int64_t count = 0; int64_t valid_count = 0; + int idx = 0; + if (node_type == "") { + VLOG(0) << "node_type not specified, loading edges to " << id_to_feature[0] + << " part"; + } else { + if (feature_to_id.find(node_type) == feature_to_id.end()) { + VLOG(0) << "node_type " << node_type + << " is not defined, nothing will be loaded"; + return 0; + } + idx = feature_to_id[node_type]; + } for (auto path : paths) { std::ifstream file(path); std::string line; @@ -650,12 +667,12 @@ int32_t GraphTable::load_nodes(const std::string &path, std::string node_type) { size_t index = shard_id - shard_start; - auto node = shards[index]->add_feature_node(id); - - node->set_feature_size(feat_name.size()); + // auto node = shards[index]->add_feature_node(id); + auto node = feature_shards[idx][index]->add_feature_node(id); + node->set_feature_size(feat_name[idx].size()); for (size_t slice = 2; slice < values.size(); slice++) { - auto feat = this->parse_feature(values[slice]); + auto feat = this->parse_feature(idx, values[slice]); if (feat.first >= 0) { node->set_feature(feat.first, feat.second); } else { @@ -672,16 +689,37 @@ int32_t GraphTable::load_nodes(const std::string &path, std::string node_type) { return 0; } -int32_t GraphTable::load_edges(const std::string &path, bool reverse_edge) { +int32_t GraphTable::build_sampler(int idx, std::string sample_type) { + for (auto &shard : edge_shards[idx]) { + auto bucket = shard->get_bucket(); + for (size_t i = 0; i < bucket.size(); i++) { + bucket[i]->build_sampler(sample_type); + } + } + return 0; +} +int32_t GraphTable::load_edges(const std::string &path, bool reverse_edge, + const std::string &edge_type) { // #ifdef PADDLE_WITH_HETERPS // if (gpups_mode) pthread_rwlock_rdlock(rw_lock.get()); // #endif + int idx = 0; + if (edge_type == "") { + VLOG(0) << "edge_type not specified, loading edges to " << id_to_edge[0] + << " part"; + } else { + if (edge_to_id.find(edge_type) == edge_to_id.end()) { + VLOG(0) << "edge_type " << edge_type + << " is not defined, nothing will be loaded"; + return 0; + } + idx = edge_to_id[edge_type]; + } auto paths = paddle::string::split_string(path, ";"); int64_t count = 0; std::string sample_type = "random"; bool is_weighted = false; int valid_count = 0; - int extra_alloc_index = 0; for (auto path : paths) { std::ifstream file(path); std::string line; @@ -704,195 +742,68 @@ int32_t GraphTable::load_edges(const std::string &path, bool reverse_edge) { size_t src_shard_id = src_id % shard_num; if (src_shard_id >= shard_end || src_shard_id < shard_start) { - if (use_duplicate_nodes == false || - extra_nodes.find(src_id) == extra_nodes.end()) { - VLOG(4) << "will not load " << src_id << " from " << path - << ", please check id distribution"; - continue; - } - int index; - if (extra_nodes_to_thread_index.find(src_id) != - extra_nodes_to_thread_index.end()) { - index = extra_nodes_to_thread_index[src_id]; - } else { - index = extra_alloc_index++; - extra_alloc_index %= task_pool_size_; - extra_nodes_to_thread_index[src_id] = index; - } - extra_shards[index]->add_graph_node(src_id)->build_edges(is_weighted); - extra_shards[index]->add_neighbor(src_id, dst_id, weight); - valid_count++; + VLOG(4) << "will not load " << src_id << " from " << path + << ", please check id distribution"; continue; } + if (count % 1000000 == 0) { VLOG(0) << count << " edges are loaded from filepath"; VLOG(0) << line; } size_t index = src_shard_id - shard_start; - shards[index]->add_graph_node(src_id)->build_edges(is_weighted); - shards[index]->add_neighbor(src_id, dst_id, weight); + edge_shards[idx][index]->add_graph_node(src_id)->build_edges(is_weighted); + edge_shards[idx][index]->add_neighbor(src_id, dst_id, weight); valid_count++; } } VLOG(0) << valid_count << "/" << count << " edges are loaded successfully in " << path; - std::vector used(task_pool_size_, 0); // Build Sampler j - for (auto &shard : shards) { - auto bucket = shard->get_bucket(); - for (size_t i = 0; i < bucket.size(); i++) { - bucket[i]->build_sampler(sample_type); - used[get_thread_pool_index(bucket[i]->get_id())]++; - } - } - /*----------------------- - relocate the duplicate nodes to make them distributed evenly among threads. -*/ - if (!use_duplicate_nodes) { - // #ifdef PADDLE_WITH_HETERPS - // if (gpups_mode) pthread_rwlock_unlock(rw_lock.get()); - // #endif - - return 0; - } - for (auto &shard : extra_shards) { + for (auto &shard : edge_shards[idx]) { auto bucket = shard->get_bucket(); for (size_t i = 0; i < bucket.size(); i++) { bucket[i]->build_sampler(sample_type); } } - int size = extra_nodes_to_thread_index.size(); - if (size == 0) return 0; - std::vector index; - for (int i = 0; i < (int)used.size(); i++) index.push_back(i); - sort(index.begin(), index.end(), - [&](int &a, int &b) { return used[a] < used[b]; }); - std::vector alloc(index.size(), 0), has_alloc(index.size(), 0); - int t = 1, aim = 0, mod = 0; - for (; t < (int)used.size(); t++) { - if ((used[index[t]] - used[index[t - 1]]) * t >= size) { - break; - } else { - size -= (used[index[t]] - used[index[t - 1]]) * t; - } - } - aim = used[index[t - 1]] + size / t; - mod = size % t; - for (int x = t - 1; x >= 0; x--) { - alloc[index[x]] = aim; - if (t - x <= mod) alloc[index[x]]++; - alloc[index[x]] -= used[index[x]]; - } - std::vector vec[index.size()]; - for (auto p : extra_nodes_to_thread_index) { - has_alloc[p.second]++; - vec[p.second].push_back(p.first); - } - sort(index.begin(), index.end(), [&](int &a, int &b) { - return has_alloc[a] - alloc[a] < has_alloc[b] - alloc[b]; - }); - int left = 0, right = (int)index.size() - 1; - while (left < right) { - if (has_alloc[index[right]] - alloc[index[right]] == 0) break; - int x = std::min(alloc[index[left]] - has_alloc[index[left]], - has_alloc[index[right]] - alloc[index[right]]); - has_alloc[index[left]] += x; - has_alloc[index[right]] -= x; - int64_t id; - while (x--) { - id = vec[index[right]].back(); - vec[index[right]].pop_back(); - extra_nodes_to_thread_index[id] = index[left]; - vec[index[left]].push_back(id); - } - if (has_alloc[index[right]] - alloc[index[right]] == 0) right--; - if (alloc[index[left]] - has_alloc[index[left]] == 0) left++; - } - std::vector extra_shards_copy; - for (int i = 0; i < task_pool_size_; ++i) { - extra_shards_copy.push_back(new GraphShard()); - } - for (auto &shard : extra_shards) { - auto &bucket = shard->get_bucket(); - auto &node_location = shard->get_node_location(); - while (bucket.size()) { - Node *temp = bucket.back(); - bucket.pop_back(); - node_location.erase(temp->get_id()); - extra_shards_copy[extra_nodes_to_thread_index[temp->get_id()]] - ->add_graph_node(temp); - } - } - for (int i = 0; i < task_pool_size_; ++i) { - delete extra_shards[i]; - extra_shards[i] = extra_shards_copy[i]; - } - // #ifdef PADDLE_WITH_HETERPS - // if (gpups_mode) pthread_rwlock_unlock(rw_lock.get()); - // #endif return 0; } -Node *GraphTable::find_node(int64_t id) { +Node *GraphTable::find_node(int type_id, int idx, int64_t id) { size_t shard_id = id % shard_num; if (shard_id >= shard_end || shard_id < shard_start) { - if (use_duplicate_nodes == false || extra_nodes_to_thread_index.size() == 0) - return nullptr; - auto iter = extra_nodes_to_thread_index.find(id); - if (iter == extra_nodes_to_thread_index.end()) - return nullptr; - else { - return extra_shards[iter->second]->find_node(id); - } + return nullptr; } size_t index = shard_id - shard_start; - Node *node = shards[index]->find_node(id); + auto &search_shards = type_id == 0 ? edge_shards[idx] : feature_shards[idx]; + Node *node = search_shards[index]->find_node(id); return node; } uint32_t GraphTable::get_thread_pool_index(int64_t node_id) { - if (use_duplicate_nodes == false || extra_nodes_to_thread_index.size() == 0) - return node_id % shard_num % shard_num_per_server % task_pool_size_; - size_t src_shard_id = node_id % shard_num; - if (src_shard_id >= shard_end || src_shard_id < shard_start) { - auto iter = extra_nodes_to_thread_index.find(node_id); - if (iter != extra_nodes_to_thread_index.end()) { - return iter->second; - } - } - return src_shard_id % shard_num_per_server % task_pool_size_; + return node_id % shard_num % shard_num_per_server % task_pool_size_; } uint32_t GraphTable::get_thread_pool_index_by_shard_index(int64_t shard_index) { return shard_index % shard_num_per_server % task_pool_size_; } -int32_t GraphTable::clear_nodes() { - std::vector> tasks; - for (size_t i = 0; i < shards.size(); i++) { - tasks.push_back( - _shards_task_pool[i % task_pool_size_]->enqueue([this, i]() -> int { - this->shards[i]->clear(); - return 0; - })); - } - for (size_t i = 0; i < extra_shards.size(); i++) { - tasks.push_back(_shards_task_pool[i]->enqueue([this, i]() -> int { - this->extra_shards[i]->clear(); - return 0; - })); +int32_t GraphTable::clear_nodes(int type_id, int idx) { + auto &search_shards = type_id == 0 ? edge_shards[idx] : feature_shards[idx]; + for (int i = 0; i < search_shards.size(); i++) { + search_shards[i]->clear(); } - for (size_t i = 0; i < tasks.size(); i++) tasks[i].get(); return 0; } -int32_t GraphTable::random_sample_nodes(int sample_size, +int32_t GraphTable::random_sample_nodes(int type_id, int idx, int sample_size, std::unique_ptr &buffer, int &actual_size) { int total_size = 0; + auto &shards = type_id == 0 ? edge_shards[idx] : feature_shards[idx]; for (int i = 0; i < (int)shards.size(); i++) { total_size += shards[i]->get_size(); } @@ -947,7 +858,7 @@ int32_t GraphTable::random_sample_nodes(int sample_size, } for (auto &pair : first_half) second_half.push_back(pair); std::vector res; - get_nodes_ids_by_ranges(second_half, res); + get_nodes_ids_by_ranges(type_id, idx, second_half, res); actual_size = res.size() * sizeof(int64_t); buffer.reset(new char[actual_size]); char *pointer = buffer.get(); @@ -955,7 +866,7 @@ int32_t GraphTable::random_sample_nodes(int sample_size, return 0; } int32_t GraphTable::random_sample_neighbors( - int64_t *node_ids, int sample_size, + int idx, int64_t *node_ids, int sample_size, std::vector> &buffers, std::vector &actual_sizes, bool need_weight) { size_t node_num = buffers.size(); @@ -964,11 +875,12 @@ int32_t GraphTable::random_sample_neighbors( std::vector> seq_id(task_pool_size_); std::vector> id_list(task_pool_size_); size_t index; - for (size_t idx = 0; idx < node_num; ++idx) { - index = get_thread_pool_index(node_ids[idx]); - seq_id[index].emplace_back(idx); - id_list[index].emplace_back(node_ids[idx], sample_size, need_weight); + for (size_t idy = 0; idy < node_num; ++idy) { + index = get_thread_pool_index(node_ids[idy]); + seq_id[index].emplace_back(idy); + id_list[index].emplace_back(idx, node_ids[idy], sample_size, need_weight); } + for (int i = 0; i < (int)seq_id.size(); i++) { if (seq_id[i].size() == 0) continue; tasks.push_back(_shards_task_pool[i]->enqueue([&, i, this]() -> int { @@ -987,20 +899,20 @@ int32_t GraphTable::random_sample_neighbors( for (size_t k = 0; k < id_list[i].size(); k++) { if (index < (int)r.size() && r[index].first.node_key == id_list[i][k].node_key) { - idx = seq_id[i][k]; - actual_sizes[idx] = r[index].second.actual_size; - buffers[idx] = r[index].second.buffer; + int idy = seq_id[i][k]; + actual_sizes[idy] = r[index].second.actual_size; + buffers[idy] = r[index].second.buffer; index++; } else { node_id = id_list[i][k].node_key; - Node *node = find_node(node_id); - idx = seq_id[i][k]; - int &actual_size = actual_sizes[idx]; + Node *node = find_node(0, idx, node_id); + int idy = seq_id[i][k]; + int &actual_size = actual_sizes[idy]; if (node == nullptr) { #ifdef PADDLE_WITH_HETERPS if (search_level == 2) { char *buffer_addr = random_sample_neighbor_from_ssd( - node_id, sample_size, rng, actual_size); + idx, node_id, sample_size, rng, actual_size); if (actual_size != 0) { std::shared_ptr &buffer = buffers[idx]; buffer.reset(buffer_addr, char_del); @@ -1011,7 +923,7 @@ int32_t GraphTable::random_sample_neighbors( actual_size = 0; continue; } - std::shared_ptr &buffer = buffers[idx]; + std::shared_ptr &buffer = buffers[idy]; std::vector res = node->sample_k(sample_size, rng); actual_size = res.size() * (need_weight ? (Node::id_size + Node::weight_size) @@ -1021,7 +933,7 @@ int32_t GraphTable::random_sample_neighbors( float weight; char *buffer_addr = new char[actual_size]; if (response == LRUResponse::ok) { - sample_keys.emplace_back(node_id, sample_size, need_weight); + sample_keys.emplace_back(idx, node_id, sample_size, need_weight); sample_res.emplace_back(actual_size, buffer_addr); buffer = sample_res.back().buffer; } else { @@ -1052,16 +964,16 @@ int32_t GraphTable::random_sample_neighbors( return 0; } -int32_t GraphTable::get_node_feat(const std::vector &node_ids, +int32_t GraphTable::get_node_feat(int idx, const std::vector &node_ids, const std::vector &feature_names, std::vector> &res) { size_t node_num = node_ids.size(); std::vector> tasks; - for (size_t idx = 0; idx < node_num; ++idx) { - int64_t node_id = node_ids[idx]; + for (size_t idy = 0; idy < node_num; ++idy) { + int64_t node_id = node_ids[idy]; tasks.push_back(_shards_task_pool[get_thread_pool_index(node_id)]->enqueue( - [&, idx, node_id]() -> int { - Node *node = find_node(node_id); + [&, idx, idy, node_id]() -> int { + Node *node = find_node(1, idx, node_id); if (node == nullptr) { return 0; @@ -1069,59 +981,61 @@ int32_t GraphTable::get_node_feat(const std::vector &node_ids, for (int feat_idx = 0; feat_idx < (int)feature_names.size(); ++feat_idx) { const std::string &feature_name = feature_names[feat_idx]; - if (feat_id_map.find(feature_name) != feat_id_map.end()) { + if (feat_id_map[idx].find(feature_name) != feat_id_map[idx].end()) { // res[feat_idx][idx] = // node->get_feature(feat_id_map[feature_name]); - auto feat = node->get_feature(feat_id_map[feature_name]); - res[feat_idx][idx] = feat; + auto feat = node->get_feature(feat_id_map[idx][feature_name]); + res[feat_idx][idy] = feat; } } return 0; })); } - for (size_t idx = 0; idx < node_num; ++idx) { - tasks[idx].get(); + for (size_t idy = 0; idy < node_num; ++idy) { + tasks[idy].get(); } return 0; } int32_t GraphTable::set_node_feat( - const std::vector &node_ids, + int idx, const std::vector &node_ids, const std::vector &feature_names, const std::vector> &res) { size_t node_num = node_ids.size(); std::vector> tasks; - for (size_t idx = 0; idx < node_num; ++idx) { - int64_t node_id = node_ids[idx]; + for (size_t idy = 0; idy < node_num; ++idy) { + int64_t node_id = node_ids[idy]; tasks.push_back(_shards_task_pool[get_thread_pool_index(node_id)]->enqueue( - [&, idx, node_id]() -> int { + [&, idx, idy, node_id]() -> int { size_t index = node_id % this->shard_num - this->shard_start; - auto node = shards[index]->add_feature_node(node_id); - node->set_feature_size(this->feat_name.size()); + auto node = feature_shards[idx][index]->add_feature_node(node_id); + node->set_feature_size(this->feat_name[idx].size()); for (int feat_idx = 0; feat_idx < (int)feature_names.size(); ++feat_idx) { const std::string &feature_name = feature_names[feat_idx]; - if (feat_id_map.find(feature_name) != feat_id_map.end()) { - node->set_feature(feat_id_map[feature_name], res[feat_idx][idx]); + if (feat_id_map[idx].find(feature_name) != feat_id_map[idx].end()) { + node->set_feature(feat_id_map[idx][feature_name], + res[feat_idx][idy]); } } return 0; })); } - for (size_t idx = 0; idx < node_num; ++idx) { - tasks[idx].get(); + for (size_t idy = 0; idy < node_num; ++idy) { + tasks[idy].get(); } return 0; } std::pair GraphTable::parse_feature( - std::string feat_str) { + int idx, std::string feat_str) { // Return (feat_id, btyes) if name are in this->feat_name, else return (-1, // "") auto fields = paddle::string::split_string(feat_str, " "); - if (this->feat_id_map.count(fields[0])) { - int32_t id = this->feat_id_map[fields[0]]; - std::string dtype = this->feat_dtype[id]; + if (feat_id_map[idx].count(fields[0])) { + // if (this->feat_id_map.count(fields[0])) { + int32_t id = this->feat_id_map[idx][fields[0]]; + std::string dtype = this->feat_dtype[idx][id]; std::vector values(fields.begin() + 1, fields.end()); if (dtype == "feasign") { return std::make_pair( @@ -1146,15 +1060,17 @@ std::pair GraphTable::parse_feature( return std::make_pair(-1, ""); } -int32_t GraphTable::pull_graph_list(int start, int total_size, +int32_t GraphTable::pull_graph_list(int type_id, int idx, int start, + int total_size, std::unique_ptr &buffer, int &actual_size, bool need_feature, int step) { if (start < 0) start = 0; int size = 0, cur_size; + auto &search_shards = type_id == 0 ? edge_shards[idx] : feature_shards[idx]; std::vector>> tasks; - for (size_t i = 0; i < shards.size() && total_size > 0; i++) { - cur_size = shards[i]->get_size(); + for (size_t i = 0; i < search_shards.size() && total_size > 0; i++) { + cur_size = search_shards[i]->get_size(); if (size + cur_size <= start) { size += cur_size; continue; @@ -1162,8 +1078,9 @@ int32_t GraphTable::pull_graph_list(int start, int total_size, int count = std::min(1 + (size + cur_size - start - 1) / step, total_size); int end = start + (count - 1) * step + 1; tasks.push_back(_shards_task_pool[i % task_pool_size_]->enqueue( - [this, i, start, end, step, size]() -> std::vector { - return this->shards[i]->get_batch(start - size, end - size, step); + [&search_shards, this, i, start, end, step, + size]() -> std::vector { + return search_shards[i]->get_batch(start - size, end - size, step); })); start += count * step; total_size -= count; @@ -1250,6 +1167,41 @@ int32_t GraphTable::Initialize(const GraphParameter &graph) { _shards_task_rng_pool.push_back(paddle::framework::GetCPURandomEngine(0)); } auto graph_feature = graph.graph_feature(); + auto node_types = graph.node_types(); + auto edge_types = graph.edge_types(); + VLOG(0) << "got " << edge_types.size() << "edge types in total"; + feat_id_map.resize(node_types.size()); + for (int k = 0; k < edge_types.size(); k++) { + VLOG(0) << "in initialize: get a edge_type " << edge_types[k]; + edge_to_id[edge_types[k]] = k; + id_to_edge.push_back(edge_types[k]); + } + feat_name.resize(node_types.size()); + feat_shape.resize(node_types.size()); + feat_dtype.resize(node_types.size()); + VLOG(0) << "got " << node_types.size() << "node types in total"; + for (int k = 0; k < node_types.size(); k++) { + feature_to_id[node_types[k]] = k; + auto node_type = node_types[k]; + auto feature = graph_feature[k]; + id_to_feature.push_back(node_type); + int feat_conf_size = static_cast(feature.name().size()); + + for (int i = 0; i < feat_conf_size; i++) { + // auto &f_name = common.attributes()[i]; + // auto &f_shape = common.dims()[i]; + // auto &f_dtype = common.params()[i]; + auto &f_name = feature.name()[i]; + auto &f_shape = feature.shape()[i]; + auto &f_dtype = feature.dtype()[i]; + feat_name[k].push_back(f_name); + feat_shape[k].push_back(f_shape); + feat_dtype[k].push_back(f_dtype); + feat_id_map[k][f_name] = i; + VLOG(0) << "init graph table feat conf name:" << f_name + << " shape:" << f_shape << " dtype:" << f_dtype; + } + } // this->table_name = common.table_name(); // this->table_type = common.name(); this->table_name = graph.table_name(); @@ -1257,21 +1209,7 @@ int32_t GraphTable::Initialize(const GraphParameter &graph) { VLOG(0) << " init graph table type " << this->table_type << " table name " << this->table_name; // int feat_conf_size = static_cast(common.attributes().size()); - int feat_conf_size = static_cast(graph_feature.name().size()); - for (int i = 0; i < feat_conf_size; i++) { - // auto &f_name = common.attributes()[i]; - // auto &f_shape = common.dims()[i]; - // auto &f_dtype = common.params()[i]; - auto &f_name = graph_feature.name()[i]; - auto &f_shape = graph_feature.shape()[i]; - auto &f_dtype = graph_feature.dtype()[i]; - this->feat_name.push_back(f_name); - this->feat_shape.push_back(f_shape); - this->feat_dtype.push_back(f_dtype); - this->feat_id_map[f_name] = i; - VLOG(0) << "init graph table feat conf name:" << f_name - << " shape:" << f_shape << " dtype:" << f_dtype; - } + // int feat_conf_size = static_cast(graph_feature.name().size()); VLOG(0) << "in init graph table shard num = " << shard_num << " shard_idx" << _shard_idx; shard_num_per_server = sparse_local_shard_num(shard_num, server_num); @@ -1279,12 +1217,17 @@ int32_t GraphTable::Initialize(const GraphParameter &graph) { shard_end = shard_start + shard_num_per_server; VLOG(0) << "in init graph table shard idx = " << _shard_idx << " shard_start " << shard_start << " shard_end " << shard_end; - for (size_t i = 0; i < shard_num_per_server; i++) { - shards.push_back(new GraphShard()); + edge_shards.resize(id_to_edge.size()); + for (int k = 0; k < (int)edge_shards.size(); k++) { + for (size_t i = 0; i < shard_num_per_server; i++) { + edge_shards[k].push_back(new GraphShard()); + } } - use_duplicate_nodes = false; - for (int i = 0; i < task_pool_size_; i++) { - extra_shards.push_back(new GraphShard()); + feature_shards.resize(id_to_feature.size()); + for (int k = 0; k < (int)feature_shards.size(); k++) { + for (size_t i = 0; i < shard_num_per_server; i++) { + feature_shards[k].push_back(new GraphShard()); + } } return 0; diff --git a/paddle/fluid/distributed/ps/table/common_graph_table.h b/paddle/fluid/distributed/ps/table/common_graph_table.h index 863c397b08ad2..059bcb09a0a6e 100644 --- a/paddle/fluid/distributed/ps/table/common_graph_table.h +++ b/paddle/fluid/distributed/ps/table/common_graph_table.h @@ -38,13 +38,13 @@ #include #include "paddle/fluid/distributed/ps/table/accessor.h" #include "paddle/fluid/distributed/ps/table/common_table.h" -#include "paddle/fluid/distributed/ps/table/depends/rocksdb_warpper.h" #include "paddle/fluid/distributed/ps/table/graph/class_macro.h" #include "paddle/fluid/distributed/ps/table/graph/graph_node.h" #include "paddle/fluid/string/string_helper.h" #include "paddle/phi/core/utils/rw_lock.h" #ifdef PADDLE_WITH_HETERPS +#include "paddle/fluid/distributed/ps/table/depends/rocksdb_warpper.h" #include "paddle/fluid/framework/fleet/heter_ps/gpu_graph_node.h" #endif namespace paddle { @@ -83,16 +83,20 @@ class GraphShard { enum LRUResponse { ok = 0, blocked = 1, err = 2 }; struct SampleKey { + int idx; int64_t node_key; size_t sample_size; bool is_weighted; - SampleKey(int64_t _node_key, size_t _sample_size, bool _is_weighted) - : node_key(_node_key), - sample_size(_sample_size), - is_weighted(_is_weighted) {} + SampleKey(int _idx, int64_t _node_key, size_t _sample_size, + bool _is_weighted) { + idx = _idx; + node_key = _node_key; + sample_size = _sample_size; + is_weighted = _is_weighted; + } bool operator==(const SampleKey &s) const { - return node_key == s.node_key && sample_size == s.sample_size && - is_weighted == s.is_weighted; + return idx == s.idx && node_key == s.node_key && + sample_size == s.sample_size && is_weighted == s.is_weighted; } }; @@ -435,44 +439,46 @@ class GraphTable : public Table { return (key % shard_num) / sparse_local_shard_num(shard_num, server_num); } - virtual int32_t pull_graph_list(int start, int size, + virtual int32_t pull_graph_list(int type_id, int idx, int start, int size, std::unique_ptr &buffer, int &actual_size, bool need_feature, int step); virtual int32_t random_sample_neighbors( - int64_t *node_ids, int sample_size, + int idx, int64_t *node_ids, int sample_size, std::vector> &buffers, std::vector &actual_sizes, bool need_weight); - int32_t random_sample_nodes(int sample_size, std::unique_ptr &buffers, + int32_t random_sample_nodes(int type_id, int idx, int sample_size, + std::unique_ptr &buffers, int &actual_sizes); virtual int32_t get_nodes_ids_by_ranges( - std::vector> ranges, std::vector &res); + int type_id, int idx, std::vector> ranges, + std::vector &res); virtual int32_t Initialize() { return 0; } virtual int32_t Initialize(const TableParameter &config, const FsClientParameter &fs_config); virtual int32_t Initialize(const GraphParameter &config); int32_t Load(const std::string &path, const std::string ¶m); - int32_t load_graph_split_config(const std::string &path); - int32_t load_edges(const std::string &path, bool reverse); + int32_t load_edges(const std::string &path, bool reverse, + const std::string &edge_type); int32_t load_nodes(const std::string &path, std::string node_type); - int32_t add_graph_node(std::vector &id_list, + int32_t add_graph_node(int idx, std::vector &id_list, std::vector &is_weight_list); - int32_t remove_graph_node(std::vector &id_list); + int32_t remove_graph_node(int idx, std::vector &id_list); int32_t get_server_index_by_id(int64_t id); - Node *find_node(int64_t id); + Node *find_node(int type_id, int idx, int64_t id); virtual int32_t Pull(TableContext &context) { return 0; } virtual int32_t Push(TableContext &context) { return 0; } - virtual int32_t clear_nodes(); + virtual int32_t clear_nodes(int type, int idx); virtual void Clear() {} virtual int32_t Flush() { return 0; } virtual int32_t Shrink(const std::string ¶m) { return 0; } @@ -494,14 +500,15 @@ class GraphTable : public Table { } virtual uint32_t get_thread_pool_index_by_shard_index(int64_t shard_index); virtual uint32_t get_thread_pool_index(int64_t node_id); - virtual std::pair parse_feature(std::string feat_str); + virtual std::pair parse_feature(int idx, + std::string feat_str); - virtual int32_t get_node_feat(const std::vector &node_ids, + virtual int32_t get_node_feat(int idx, const std::vector &node_ids, const std::vector &feature_names, std::vector> &res); virtual int32_t set_node_feat( - const std::vector &node_ids, + int idx, const std::vector &node_ids, const std::vector &feature_names, const std::vector> &res); @@ -532,24 +539,28 @@ class GraphTable : public Table { // return 0; // } virtual char *random_sample_neighbor_from_ssd( - int64_t id, int sample_size, const std::shared_ptr rng, - int &actual_size); - virtual int32_t add_node_to_ssd(int64_t id, char *data, int len); + int idx, int64_t id, int sample_size, + const std::shared_ptr rng, int &actual_size); + virtual int32_t add_node_to_ssd(int type_id, int idx, int64_t src_id, + char *data, int len); virtual paddle::framework::GpuPsCommGraph make_gpu_ps_graph( - std::vector ids); + int idx, std::vector ids); // virtual GraphSampler *get_graph_sampler() { return graph_sampler.get(); } int search_level; #endif - virtual int32_t add_comm_edge(int64_t src_id, int64_t dst_id); - std::vector shards, extra_shards; + virtual int32_t add_comm_edge(int idx, int64_t src_id, int64_t dst_id); + virtual int32_t build_sampler(int idx, std::string sample_type = "random"); + std::vector> edge_shards, feature_shards; size_t shard_start, shard_end, server_num, shard_num_per_server, shard_num; int task_pool_size_ = 24; const int random_sample_nodes_ranges = 3; - std::vector feat_name; - std::vector feat_dtype; - std::vector feat_shape; - std::unordered_map feat_id_map; + std::vector> feat_name; + std::vector> feat_dtype; + std::vector> feat_shape; + std::vector> feat_id_map; + std::unordered_map feature_to_id, edge_to_id; + std::vector id_to_feature, id_to_edge; std::string table_name; std::string table_type; @@ -624,7 +635,7 @@ namespace std { template <> struct hash { size_t operator()(const paddle::distributed::SampleKey &s) const { - return s.node_key ^ s.sample_size; + return s.idx ^ s.node_key ^ s.sample_size; } }; } diff --git a/paddle/fluid/distributed/ps/table/ctr_accessor.cc b/paddle/fluid/distributed/ps/table/ctr_accessor.cc index 715abe270e52b..ef7311824faa6 100644 --- a/paddle/fluid/distributed/ps/table/ctr_accessor.cc +++ b/paddle/fluid/distributed/ps/table/ctr_accessor.cc @@ -34,6 +34,8 @@ int CtrCommonAccessor::Initialize() { common_feature_value.embedx_dim = _config.embedx_dim(); common_feature_value.embedx_sgd_dim = _embedx_sgd_rule->Dim(); _show_click_decay_rate = _config.ctr_accessor_param().show_click_decay_rate(); + _ssd_unseenday_threshold = + _config.ctr_accessor_param().ssd_unseenday_threshold(); if (_config.ctr_accessor_param().show_scale()) { _show_scale = true; @@ -77,6 +79,25 @@ bool CtrCommonAccessor::Shrink(float* value) { return false; } +bool CtrCommonAccessor::SaveCache(float* value, int param, + double global_cache_threshold) { + auto base_threshold = _config.ctr_accessor_param().base_threshold(); + auto delta_keep_days = _config.ctr_accessor_param().delta_keep_days(); + if (ShowClickScore(common_feature_value.Show(value), + common_feature_value.Click(value)) >= base_threshold && + common_feature_value.UnseenDays(value) <= delta_keep_days) { + return common_feature_value.Show(value) > global_cache_threshold; + } + return false; +} + +bool CtrCommonAccessor::SaveSSD(float* value) { + if (common_feature_value.UnseenDays(value) > _ssd_unseenday_threshold) { + return true; + } + return false; +} + bool CtrCommonAccessor::Save(float* value, int param) { auto base_threshold = _config.ctr_accessor_param().base_threshold(); auto delta_threshold = _config.ctr_accessor_param().delta_threshold(); diff --git a/paddle/fluid/distributed/ps/table/ctr_accessor.h b/paddle/fluid/distributed/ps/table/ctr_accessor.h index a599bfca7f6d2..327c4cea760eb 100644 --- a/paddle/fluid/distributed/ps/table/ctr_accessor.h +++ b/paddle/fluid/distributed/ps/table/ctr_accessor.h @@ -148,6 +148,9 @@ class CtrCommonAccessor : public ValueAccessor { // param = 1, save delta feature // param = 2, save xbox base feature bool Save(float* value, int param) override; + bool SaveCache(float* value, int param, + double global_cache_threshold) override; + bool SaveSSD(float* value) override; // update delta_score and unseen_days after save void UpdateStatAfterSave(float* value, int param) override; // keys不存在时，为values生成随机值 diff --git a/paddle/fluid/distributed/ps/table/ctr_double_accessor.cc b/paddle/fluid/distributed/ps/table/ctr_double_accessor.cc index f0d9426343d7b..4b84b7e8c36c3 100644 --- a/paddle/fluid/distributed/ps/table/ctr_double_accessor.cc +++ b/paddle/fluid/distributed/ps/table/ctr_double_accessor.cc @@ -74,25 +74,26 @@ bool CtrDoubleAccessor::Shrink(float* value) { } return false; } + bool CtrDoubleAccessor::SaveSSD(float* value) { if (CtrDoubleFeatureValue::UnseenDays(value) > _ssd_unseenday_threshold) { return true; } return false; } -// bool CtrDoubleAccessor::save_cache( -// float* value, int param, double global_cache_threshold) { -// auto base_threshold = _config.ctr_accessor_param().base_threshold(); -// auto delta_keep_days = _config.ctr_accessor_param().delta_keep_days(); -// if (ShowClickScore(CtrDoubleFeatureValue::Show(value), -// CtrDoubleFeatureValue::Click(value)) >= base_threshold -// && CtrDoubleFeatureValue::UnseenDays(value) <= -// delta_keep_days) { -// return CtrDoubleFeatureValue::Show(value) > -// global_cache_threshold; -// } -// return false; -// } + +bool CtrDoubleAccessor::SaveCache(float* value, int param, + double global_cache_threshold) { + auto base_threshold = _config.ctr_accessor_param().base_threshold(); + auto delta_keep_days = _config.ctr_accessor_param().delta_keep_days(); + if (ShowClickScore(CtrDoubleFeatureValue::Show(value), + CtrDoubleFeatureValue::Click(value)) >= base_threshold && + CtrDoubleFeatureValue::UnseenDays(value) <= delta_keep_days) { + return CtrDoubleFeatureValue::Show(value) > global_cache_threshold; + } + return false; +} + bool CtrDoubleAccessor::Save(float* value, int param) { // auto base_threshold = _config.ctr_accessor_param().base_threshold(); // auto delta_threshold = _config.ctr_accessor_param().delta_threshold(); diff --git a/paddle/fluid/distributed/ps/table/ctr_double_accessor.h b/paddle/fluid/distributed/ps/table/ctr_double_accessor.h index c58602065036f..5b781b2621c5b 100644 --- a/paddle/fluid/distributed/ps/table/ctr_double_accessor.h +++ b/paddle/fluid/distributed/ps/table/ctr_double_accessor.h @@ -167,6 +167,8 @@ class CtrDoubleAccessor : public ValueAccessor { // param = 1, save delta feature // param = 3, save all feature with time decay virtual bool Save(float* value, int param) override; + bool SaveCache(float* value, int param, + double global_cache_threshold) override; // update delta_score and unseen_days after save virtual void UpdateStatAfterSave(float* value, int param) override; // 判断该value是否保存到ssd diff --git a/paddle/fluid/distributed/ps/table/depends/rocksdb_warpper.h b/paddle/fluid/distributed/ps/table/depends/rocksdb_warpper.h index ff2271d468e39..223c8fafd26ab 100644 --- a/paddle/fluid/distributed/ps/table/depends/rocksdb_warpper.h +++ b/paddle/fluid/distributed/ps/table/depends/rocksdb_warpper.h @@ -11,9 +11,8 @@ // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. - #pragma once -#ifdef PADDLE_WITH_HETERPS + #include #include #include @@ -154,6 +153,5 @@ class RocksDBHandler { std::vector _handles; rocksdb::DB* _db; }; -} -} -#endif +} // distributed +} // paddle diff --git a/paddle/fluid/distributed/ps/table/memory_sparse_table.cc b/paddle/fluid/distributed/ps/table/memory_sparse_table.cc index e6c52e0b9b0c8..ee6a801fa9183 100644 --- a/paddle/fluid/distributed/ps/table/memory_sparse_table.cc +++ b/paddle/fluid/distributed/ps/table/memory_sparse_table.cc @@ -23,14 +23,17 @@ #include "glog/logging.h" #include "paddle/fluid/platform/enforce.h" +DEFINE_bool(pserver_print_missed_key_num_every_push, false, + "pserver_print_missed_key_num_every_push"); +DEFINE_bool(pserver_create_value_when_push, true, + "pserver create value when push"); +DEFINE_bool(pserver_enable_create_feasign_randomly, false, + "pserver_enable_create_feasign_randomly"); +DEFINE_int32(pserver_table_save_max_retry, 3, "pserver_table_save_max_retry"); + namespace paddle { namespace distributed { -// TODO(zhaocaibei123): configure -bool FLAGS_pserver_create_value_when_push = true; -int FLAGS_pserver_table_save_max_retry = 3; -bool FLAGS_pserver_enable_create_feasign_randomly = false; - int32_t MemorySparseTable::Initialize() { _shards_task_pool.resize(_task_pool_size); for (int i = 0; i < _shards_task_pool.size(); ++i) { @@ -142,7 +145,7 @@ int32_t MemorySparseTable::Load(const std::string& path, LOG(ERROR) << "MemorySparseTable load failed, retry it! path:" << channel_config.path << " , retry_num=" << retry_num; } - if (retry_num > paddle::distributed::FLAGS_pserver_table_save_max_retry) { + if (retry_num > FLAGS_pserver_table_save_max_retry) { LOG(ERROR) << "MemorySparseTable load failed reach max limit!"; exit(-1); } @@ -213,7 +216,7 @@ int32_t MemorySparseTable::LoadLocalFS(const std::string& path, << file_list[file_start_idx + i] << " , retry_num=" << retry_num; } - if (retry_num > paddle::distributed::FLAGS_pserver_table_save_max_retry) { + if (retry_num > FLAGS_pserver_table_save_max_retry) { LOG(ERROR) << "MemorySparseTable load failed reach max limit!"; exit(-1); } @@ -293,7 +296,7 @@ int32_t MemorySparseTable::Save(const std::string& dirname, if (is_write_failed) { _afs_client.remove(channel_config.path); } - if (retry_num > paddle::distributed::FLAGS_pserver_table_save_max_retry) { + if (retry_num > FLAGS_pserver_table_save_max_retry) { LOG(ERROR) << "MemorySparseTable save prefix failed reach max limit!"; exit(-1); } diff --git a/paddle/fluid/distributed/ps/table/memory_sparse_table.h b/paddle/fluid/distributed/ps/table/memory_sparse_table.h index 87a73bd22fa2f..6516c75a5d696 100644 --- a/paddle/fluid/distributed/ps/table/memory_sparse_table.h +++ b/paddle/fluid/distributed/ps/table/memory_sparse_table.h @@ -62,9 +62,11 @@ class MemorySparseTable : public Table { int32_t InitializeShard() override { return 0; } int32_t InitializeValue(); - int32_t Load(const std::string& path, const std::string& param) override; + virtual int32_t Load(const std::string& path, + const std::string& param) override; - int32_t Save(const std::string& path, const std::string& param) override; + virtual int32_t Save(const std::string& path, + const std::string& param) override; int32_t LoadLocalFS(const std::string& path, const std::string& param); int32_t SaveLocalFS(const std::string& path, const std::string& param, @@ -83,7 +85,7 @@ class MemorySparseTable : public Table { int32_t PushSparse(const uint64_t* keys, const float** values, size_t num); int32_t Flush() override; - int32_t Shrink(const std::string& param) override; + virtual int32_t Shrink(const std::string& param) override; void Clear() override; void* GetShard(size_t shard_idx) override { @@ -92,9 +94,9 @@ class MemorySparseTable : public Table { protected: const int _task_pool_size = 24; - size_t _avg_local_shard_num; - size_t _real_local_shard_num; - size_t _sparse_table_shard_num; + int _avg_local_shard_num; + int _real_local_shard_num; + int _sparse_table_shard_num; std::vector> _shards_task_pool; std::unique_ptr _local_shards; }; diff --git a/paddle/fluid/distributed/ps/table/sparse_accessor.h b/paddle/fluid/distributed/ps/table/sparse_accessor.h index 5ca5d21707a2b..875904847b2ea 100644 --- a/paddle/fluid/distributed/ps/table/sparse_accessor.h +++ b/paddle/fluid/distributed/ps/table/sparse_accessor.h @@ -135,6 +135,11 @@ class SparseAccessor : public ValueAccessor { // param = 1, save delta feature // param = 2, save xbox base feature bool Save(float* value, int param) override; + + bool SaveCache(float* value, int param, double global_cache_threshold) { + return false; + } + bool SaveSSD(float* value) { return false; } // update delta_score and unseen_days after save void UpdateStatAfterSave(float* value, int param) override; // keys不存在时，为values生成随机值 diff --git a/paddle/fluid/distributed/ps/table/ssd_sparse_table.cc b/paddle/fluid/distributed/ps/table/ssd_sparse_table.cc new file mode 100644 index 0000000000000..b1359d1323d89 --- /dev/null +++ b/paddle/fluid/distributed/ps/table/ssd_sparse_table.cc @@ -0,0 +1,759 @@ +// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "paddle/fluid/distributed/ps/table/ssd_sparse_table.h" +#include "paddle/fluid/distributed/common/cost_timer.h" +#include "paddle/fluid/distributed/common/local_random.h" +#include "paddle/fluid/distributed/common/topk_calculator.h" +#include "paddle/fluid/framework/archive.h" +#include "paddle/utils/string/string_helper.h" + +DECLARE_bool(pserver_print_missed_key_num_every_push); +DECLARE_bool(pserver_create_value_when_push); +DECLARE_bool(pserver_enable_create_feasign_randomly); +DEFINE_bool(pserver_open_strict_check, false, "pserver_open_strict_check"); +DEFINE_string(rocksdb_path, "database", "path of sparse table rocksdb file"); +DEFINE_int32(pserver_load_batch_size, 5000, "load batch size for ssd"); + +namespace paddle { +namespace distributed { + +int32_t SSDSparseTable::Initialize() { + MemorySparseTable::Initialize(); + _db = paddle::distributed::RocksDBHandler::GetInstance(); + _db->initialize(FLAGS_rocksdb_path, _real_local_shard_num); + return 0; +} + +int32_t SSDSparseTable::InitializeShard() { return 0; } + +int32_t SSDSparseTable::PullSparse(float* pull_values, const uint64_t* keys, + size_t num) { + CostTimer timer("pserver_downpour_sparse_select_all"); + size_t value_size = _value_accesor->GetAccessorInfo().size / sizeof(float); + size_t mf_value_size = + _value_accesor->GetAccessorInfo().mf_size / sizeof(float); + size_t select_value_size = + _value_accesor->GetAccessorInfo().select_size / sizeof(float); + + { // 从table取值 or create + std::vector> tasks(_real_local_shard_num); + std::vector>> task_keys( + _real_local_shard_num); + for (size_t i = 0; i < num; ++i) { + int shard_id = (keys[i] % _sparse_table_shard_num) % _avg_local_shard_num; + task_keys[shard_id].push_back({keys[i], i}); + } + + std::atomic missed_keys{0}; + for (size_t shard_id = 0; shard_id < _real_local_shard_num; ++shard_id) { + tasks[shard_id] = + _shards_task_pool[shard_id % _shards_task_pool.size()]->enqueue( + [this, shard_id, &task_keys, value_size, mf_value_size, + select_value_size, pull_values, keys, &missed_keys]() -> int { + auto& keys = task_keys[shard_id]; + auto& local_shard = _local_shards[shard_id]; + float data_buffer[value_size]; + float* data_buffer_ptr = data_buffer; + for (int i = 0; i < keys.size(); ++i) { + uint64_t key = keys[i].first; + auto itr = local_shard.find(key); + size_t data_size = value_size - mf_value_size; + if (itr == local_shard.end()) { + // pull rocksdb + std::string tmp_string(""); + if (_db->get(shard_id, (char*)&key, sizeof(uint64_t), + tmp_string) > 0) { + ++missed_keys; + if (FLAGS_pserver_create_value_when_push) { + memset(data_buffer, 0, sizeof(float) * data_size); + } else { + auto& feature_value = local_shard[key]; + feature_value.resize(data_size); + float* data_ptr = + const_cast(feature_value.data()); + _value_accesor->Create(&data_buffer_ptr, 1); + memcpy(data_ptr, data_buffer_ptr, + data_size * sizeof(float)); + } + } else { + data_size = tmp_string.size() / sizeof(float); + memcpy(data_buffer_ptr, + paddle::string::str_to_float(tmp_string), + data_size * sizeof(float)); + // from rocksdb to mem + auto& feature_value = local_shard[key]; + feature_value.resize(data_size); + memcpy(const_cast(feature_value.data()), + data_buffer_ptr, data_size * sizeof(float)); + _db->del_data(shard_id, (char*)&key, sizeof(uint64_t)); + } + } else { + data_size = itr.value().size(); + memcpy(data_buffer_ptr, itr.value().data(), + data_size * sizeof(float)); + } + for (int mf_idx = data_size; mf_idx < value_size; ++mf_idx) { + data_buffer[mf_idx] = 0.0; + } + int pull_data_idx = keys[i].second; + float* select_data = + pull_values + pull_data_idx * select_value_size; + _value_accesor->Select(&select_data, + (const float**)&data_buffer_ptr, 1); + } + return 0; + }); + } + for (size_t i = 0; i < _real_local_shard_num; ++i) { + tasks[i].wait(); + } + if (FLAGS_pserver_print_missed_key_num_every_push) { + LOG(WARNING) << "total pull keys:" << num + << " missed_keys:" << missed_keys.load(); + } + } + return 0; +} + +int32_t SSDSparseTable::PushSparse(const uint64_t* keys, const float* values, + size_t num) { + CostTimer timer("pserver_downpour_sparse_update_all"); + // 构造value push_value的数据指针 + size_t value_col = _value_accesor->GetAccessorInfo().size / sizeof(float); + size_t mf_value_col = + _value_accesor->GetAccessorInfo().mf_size / sizeof(float); + size_t update_value_col = + _value_accesor->GetAccessorInfo().update_size / sizeof(float); + { + std::vector> tasks(_real_local_shard_num); + std::vector>> task_keys( + _real_local_shard_num); + for (size_t i = 0; i < num; ++i) { + int shard_id = (keys[i] % _sparse_table_shard_num) % _avg_local_shard_num; + task_keys[shard_id].push_back({keys[i], i}); + } + for (size_t shard_id = 0; shard_id < _real_local_shard_num; ++shard_id) { + tasks[shard_id] = + _shards_task_pool[shard_id % _shards_task_pool.size()]->enqueue( + [this, shard_id, value_col, mf_value_col, update_value_col, + values, &task_keys]() -> int { + auto& keys = task_keys[shard_id]; + auto& local_shard = _local_shards[shard_id]; + float data_buffer[value_col]; + float* data_buffer_ptr = data_buffer; + for (int i = 0; i < keys.size(); ++i) { + uint64_t key = keys[i].first; + uint64_t push_data_idx = keys[i].second; + const float* update_data = + values + push_data_idx * update_value_col; + auto itr = local_shard.find(key); + if (itr == local_shard.end()) { + if (FLAGS_pserver_enable_create_feasign_randomly && + !_value_accesor->CreateValue(1, update_data)) { + continue; + } + auto value_size = value_col - mf_value_col; + auto& feature_value = local_shard[key]; + feature_value.resize(value_size); + _value_accesor->Create(&data_buffer_ptr, 1); + memcpy(const_cast(feature_value.data()), + data_buffer_ptr, value_size * sizeof(float)); + itr = local_shard.find(key); + } + auto& feature_value = itr.value(); + float* value_data = const_cast(feature_value.data()); + size_t value_size = feature_value.size(); + + if (value_size == + value_col) { // 已拓展到最大size, 则就地update + _value_accesor->Update(&value_data, &update_data, 1); + } else { // 拷入buffer区进行update，然后再回填，不需要的mf则回填时抛弃了 + memcpy(data_buffer_ptr, value_data, + value_size * sizeof(float)); + _value_accesor->Update(&data_buffer_ptr, &update_data, 1); + if (_value_accesor->NeedExtendMF(data_buffer)) { + feature_value.resize(value_col); + value_data = const_cast(feature_value.data()); + _value_accesor->Create(&value_data, 1); + } + memcpy(value_data, data_buffer_ptr, + value_size * sizeof(float)); + } + } + return 0; + }); + } + for (size_t i = 0; i < _real_local_shard_num; ++i) { + tasks[i].wait(); + } + } + /* + //update && value 的转置 + thread_local Eigen::MatrixXf update_matrix; + float* transposed_update_data[update_value_col]; + make_matrix_with_eigen(num, update_value_col, update_matrix, + transposed_update_data); + copy_array_to_eigen(values, update_matrix); + + thread_local Eigen::MatrixXf value_matrix; + float* transposed_value_data[value_col]; + make_matrix_with_eigen(num, value_col, value_matrix, transposed_value_data); + copy_matrix_to_eigen((const float**)(value_ptrs->data()), value_matrix); + + //批量update + { + CostTimer accessor_timer("pslib_downpour_sparse_update_accessor"); + _value_accesor->update(transposed_value_data, (const + float**)transposed_update_data, num); + } + copy_eigen_to_matrix(value_matrix, value_ptrs->data()); + */ + return 0; +} + +int32_t SSDSparseTable::Shrink(const std::string& param) { + int thread_num = _real_local_shard_num < 20 ? _real_local_shard_num : 20; + omp_set_num_threads(thread_num); +#pragma omp parallel for schedule(dynamic) + for (size_t i = 0; i < _real_local_shard_num; ++i) { + uint64_t mem_count = 0; + uint64_t ssd_count = 0; + + LOG(INFO) << "SSDSparseTable begin shrink shard:" << i; + auto& shard = _local_shards[i]; + for (auto it = shard.begin(); it != shard.end();) { + if (_value_accesor->Shrink(it.value().data())) { + it = shard.erase(it); + mem_count++; + } else { + ++it; + } + } + auto* it = _db->get_iterator(i); + for (it->SeekToFirst(); it->Valid(); it->Next()) { + if (_value_accesor->Shrink( + paddle::string::str_to_float(it->value().data()))) { + _db->del_data(i, it->key().data(), it->key().size()); + ssd_count++; + } else { + _db->put(i, it->key().data(), it->key().size(), it->value().data(), + it->value().size()); + } + } + delete it; + LOG(INFO) << "SSDSparseTable shrink success. shard:" << i << " delete MEM[" + << mem_count << "] SSD[" << ssd_count << "]"; + //_db->flush(i); + } + return 0; +} + +int32_t SSDSparseTable::UpdateTable() { + // TODO implement with multi-thread + int count = 0; + for (size_t i = 0; i < _real_local_shard_num; ++i) { + auto& shard = _local_shards[i]; + // from mem to ssd + for (auto it = shard.begin(); it != shard.end();) { + if (_value_accesor->SaveSSD(it.value().data())) { + _db->put(i, (char*)&it.key(), sizeof(uint64_t), + (char*)it.value().data(), it.value().size() * sizeof(float)); + count++; + it = shard.erase(it); + } else { + ++it; + } + } + _db->flush(i); + } + LOG(INFO) << "Table>> update count: " << count; + return 0; +} + +int64_t SSDSparseTable::LocalSize() { + int64_t local_size = 0; + for (size_t i = 0; i < _real_local_shard_num; ++i) { + local_size += _local_shards[i].size(); + } + // TODO rocksdb size + uint64_t ssd_size = 0; + // _db->get_estimate_key_num(ssd_size); + // return local_size + ssd_size; + return local_size; +} + +int32_t SSDSparseTable::Save(const std::string& path, + const std::string& param) { + if (_real_local_shard_num == 0) { + _local_show_threshold = -1; + return 0; + } + int save_param = atoi(param.c_str()); // batch_model:0 xbox:1 + // if (save_param == 5) { + // return save_patch(path, save_param); + // } + + // LOG(INFO) << "table cache rate is: " << _config.sparse_table_cache_rate(); + LOG(INFO) << "table cache rate is: " << _config.sparse_table_cache_rate(); + LOG(INFO) << "enable_sparse_table_cache: " + << _config.enable_sparse_table_cache(); + LOG(INFO) << "LocalSize: " << LocalSize(); + if (_config.enable_sparse_table_cache()) { + LOG(INFO) << "Enable sparse table cache, top n:" << _cache_tk_size; + } + _cache_tk_size = LocalSize() * _config.sparse_table_cache_rate(); + TopkCalculator tk(_real_local_shard_num, _cache_tk_size); + size_t file_start_idx = _avg_local_shard_num * _shard_idx; + std::string table_path = TableDir(path); + _afs_client.remove(paddle::string::format_string( + "%s/part-%03d-*", table_path.c_str(), _shard_idx)); + int thread_num = _real_local_shard_num < 20 ? _real_local_shard_num : 20; + + // std::atomic feasign_size; + std::atomic feasign_size_all{0}; + // feasign_size = 0; + + omp_set_num_threads(thread_num); +#pragma omp parallel for schedule(dynamic) + for (size_t i = 0; i < _real_local_shard_num; ++i) { + FsChannelConfig channel_config; + if (_config.compress_in_save() && (save_param == 0 || save_param == 3)) { + channel_config.path = paddle::string::format_string( + "%s/part-%03d-%05d.gz", table_path.c_str(), _shard_idx, + file_start_idx + i); + } else { + channel_config.path = + paddle::string::format_string("%s/part-%03d-%05d", table_path.c_str(), + _shard_idx, file_start_idx + i); + } + channel_config.converter = _value_accesor->Converter(save_param).converter; + channel_config.deconverter = + _value_accesor->Converter(save_param).deconverter; + int err_no = 0; + int retry_num = 0; + bool is_write_failed = false; + int feasign_size = 0; + auto& shard = _local_shards[i]; + do { + err_no = 0; + feasign_size = 0; + is_write_failed = false; + auto write_channel = + _afs_client.open_w(channel_config, 1024 * 1024 * 40, &err_no); + for (auto it = shard.begin(); it != shard.end(); ++it) { + if (_config.enable_sparse_table_cache() && + (save_param == 1 || save_param == 2) && + _value_accesor->Save(it.value().data(), 4)) { + // tk.push(i, it.value().data()[2]); + tk.push(i, _value_accesor->GetField(it.value().data(), "show")); + } + if (_value_accesor->Save(it.value().data(), save_param)) { + std::string format_value = _value_accesor->ParseToString( + it.value().data(), it.value().size()); + if (0 != + write_channel->write_line(paddle::string::format_string( + "%lu %s", it.key(), format_value.c_str()))) { + ++retry_num; + is_write_failed = true; + LOG(ERROR) << "SSDSparseTable save failed, retry it! path:" + << channel_config.path << ", retry_num=" << retry_num; + break; + } + ++feasign_size; + } + } + + if (err_no == -1 && !is_write_failed) { + ++retry_num; + is_write_failed = true; + LOG(ERROR) << "SSDSparseTable save failed after write, retry it! " + << "path:" << channel_config.path + << " , retry_num=" << retry_num; + } + if (is_write_failed) { + _afs_client.remove(channel_config.path); + continue; + } + + // delta and cache and revert is all in mem, base in rocksdb + if (save_param != 1) { + auto* it = _db->get_iterator(i); + for (it->SeekToFirst(); it->Valid(); it->Next()) { + bool need_save = _value_accesor->Save( + paddle::string::str_to_float(it->value().data()), save_param); + _value_accesor->UpdateStatAfterSave( + paddle::string::str_to_float(it->value().data()), save_param); + if (need_save) { + std::string format_value = _value_accesor->ParseToString( + paddle::string::str_to_float(it->value().data()), + it->value().size() / sizeof(float)); + if (0 != + write_channel->write_line(paddle::string::format_string( + "%lu %s", *((uint64_t*)const_cast(it->key().data())), + format_value.c_str()))) { + ++retry_num; + is_write_failed = true; + LOG(ERROR) << "SSDSparseTable save failed, retry it! path:" + << channel_config.path << ", retry_num=" << retry_num; + break; + } + if (save_param == 3) { + _db->put(i, it->key().data(), it->key().size(), + it->value().data(), it->value().size()); + } + ++feasign_size; + } + } + delete it; + } + + write_channel->close(); + if (err_no == -1) { + ++retry_num; + is_write_failed = true; + LOG(ERROR) << "SSDSparseTable save failed after write, retry it! " + << "path:" << channel_config.path + << " , retry_num=" << retry_num; + } + if (is_write_failed) { + _afs_client.remove(channel_config.path); + } + } while (is_write_failed); + feasign_size_all += feasign_size; + for (auto it = shard.begin(); it != shard.end(); ++it) { + _value_accesor->UpdateStatAfterSave(it.value().data(), save_param); + } + } + if (save_param == 3) { + UpdateTable(); + _cache_tk_size = LocalSize() * _config.sparse_table_cache_rate(); + LOG(INFO) << "SSDSparseTable update success."; + } + LOG(INFO) << "SSDSparseTable save success, path:" + << paddle::string::format_string("%s/%03d/part-%03d-", path.c_str(), + _config.table_id(), _shard_idx) + << " from " << file_start_idx << " to " + << file_start_idx + _real_local_shard_num - 1; + // return feasign_size_all; + _local_show_threshold = tk.top(); + LOG(INFO) << "local cache threshold: " << _local_show_threshold; + // int32 may overflow need to change return value + return 0; +} + +int64_t SSDSparseTable::CacheShuffle( + const std::string& path, const std::string& param, double cache_threshold, + std::function(int msg_type, int to_pserver_id, + std::string& msg)> + send_msg_func, + paddle::framework::Channel>& + shuffled_channel, + const std::vector& table_ptrs) { + LOG(INFO) << "cache shuffle with cache threshold: " << cache_threshold + << " param:" << param; + int save_param = atoi(param.c_str()); // batch_model:0 xbox:1 + if (!_config.enable_sparse_table_cache() || cache_threshold < 0) { + LOG(WARNING) + << "cache shuffle failed not enable table cache or cache threshold < 0 " + << _config.enable_sparse_table_cache() << " or " << cache_threshold; + // return -1; + } + int shuffle_node_num = _config.sparse_table_cache_file_num(); + LOG(INFO) << "Table>> shuffle node num is: " << shuffle_node_num; + size_t file_start_idx = _avg_local_shard_num * _shard_idx; + int thread_num = _real_local_shard_num < 20 ? _real_local_shard_num : 20; + + std::vector< + paddle::framework::ChannelWriter>> + writers(_real_local_shard_num); + std::vector>> datas( + _real_local_shard_num); + + int feasign_size = 0; + std::vector>> + tmp_channels; + for (size_t i = 0; i < _real_local_shard_num; ++i) { + tmp_channels.push_back( + paddle::framework::MakeChannel>()); + } + + omp_set_num_threads(thread_num); +#pragma omp parallel for schedule(dynamic) + for (size_t i = 0; i < _real_local_shard_num; ++i) { + paddle::framework::ChannelWriter>& writer = + writers[i]; + // std::shared_ptr>> tmp_chan = + // paddle::framework::MakeChannel>(); + writer.Reset(tmp_channels[i].get()); + + auto& shard = _local_shards[i]; + for (auto it = shard.begin(); it != shard.end(); ++it) { + if (_value_accesor->SaveCache(it.value().data(), save_param, + cache_threshold)) { + std::string format_value = + _value_accesor->ParseToString(it.value().data(), it.value().size()); + std::pair pkv(it.key(), format_value.c_str()); + writer << pkv; + ++feasign_size; + } + } + + writer.Flush(); + writer.channel()->Close(); + } + LOG(INFO) << "SSDSparseTable cache KV save success to Channel feasigh size: " + << feasign_size + << " and start sparse cache data shuffle real local shard num: " + << _real_local_shard_num; + std::vector> local_datas; + for (size_t idx_shard = 0; idx_shard < _real_local_shard_num; ++idx_shard) { + paddle::framework::ChannelWriter>& writer = + writers[idx_shard]; + auto channel = writer.channel(); + std::vector>& data = datas[idx_shard]; + std::vector ars(shuffle_node_num); + while (channel->Read(data)) { + for (auto& t : data) { + auto pserver_id = + paddle::distributed::local_random_engine()() % shuffle_node_num; + if (pserver_id != _shard_idx) { + ars[pserver_id] << t; + } else { + local_datas.emplace_back(std::move(t)); + } + } + std::vector> total_status; + std::vector send_data_size(shuffle_node_num, 0); + std::vector send_index(shuffle_node_num); + for (int i = 0; i < shuffle_node_num; ++i) { + send_index[i] = i; + } + std::random_shuffle(send_index.begin(), send_index.end()); + for (auto index = 0u; index < shuffle_node_num; ++index) { + int i = send_index[index]; + if (i == _shard_idx) { + continue; + } + if (ars[i].Length() == 0) { + continue; + } + std::string msg(ars[i].Buffer(), ars[i].Length()); + auto ret = send_msg_func(101, i, msg); + total_status.push_back(std::move(ret)); + send_data_size[i] += ars[i].Length(); + } + for (auto& t : total_status) { + t.wait(); + } + ars.clear(); + ars = std::vector(shuffle_node_num); + data = std::vector>(); + } + } + shuffled_channel->Write(std::move(local_datas)); + LOG(INFO) << "cache shuffle finished"; + return 0; +} + +int32_t SSDSparseTable::SaveCache( + const std::string& path, const std::string& param, + paddle::framework::Channel>& + shuffled_channel) { + if (_shard_idx >= _config.sparse_table_cache_file_num()) { + return 0; + } + int save_param = atoi(param.c_str()); // batch_model:0 xbox:1 + size_t file_start_idx = _avg_local_shard_num * _shard_idx; + std::string table_path = paddle::string::format_string( + "%s/%03d_cache/", path.c_str(), _config.table_id()); + _afs_client.remove(paddle::string::format_string( + "%s/part-%03d", table_path.c_str(), _shard_idx)); + uint32_t feasign_size = 0; + FsChannelConfig channel_config; + // not compress cache model + channel_config.path = paddle::string::format_string( + "%s/part-%03d", table_path.c_str(), _shard_idx); + channel_config.converter = _value_accesor->Converter(save_param).converter; + channel_config.deconverter = + _value_accesor->Converter(save_param).deconverter; + auto write_channel = _afs_client.open_w(channel_config, 1024 * 1024 * 40); + std::vector> data; + bool is_write_failed = false; + shuffled_channel->Close(); + while (shuffled_channel->Read(data)) { + for (auto& t : data) { + ++feasign_size; + if (0 != + write_channel->write_line(paddle::string::format_string( + "%lu %s", t.first, t.second.c_str()))) { + LOG(ERROR) << "Cache Table save failed, " + "path:" + << channel_config.path << ", retry it!"; + is_write_failed = true; + break; + } + } + data = std::vector>(); + } + if (is_write_failed) { + _afs_client.remove(channel_config.path); + } + write_channel->close(); + LOG(INFO) << "SSDSparseTable cache save success, feasign: " << feasign_size + << ", path: " << channel_config.path; + shuffled_channel->Open(); + return feasign_size; +} + +int32_t SSDSparseTable::Load(const std::string& path, + const std::string& param) { + return MemorySparseTable::Load(path, param); +} + +//加载path目录下数据[start_idx, end_idx) +int32_t SSDSparseTable::Load(size_t start_idx, size_t end_idx, + const std::vector& file_list, + const std::string& param) { + if (start_idx >= file_list.size()) { + return 0; + } + int load_param = atoi(param.c_str()); + size_t feature_value_size = + _value_accesor->GetAccessorInfo().size / sizeof(float); + size_t mf_value_size = + _value_accesor->GetAccessorInfo().mf_size / sizeof(float); + + end_idx = + end_idx < _sparse_table_shard_num ? end_idx : _sparse_table_shard_num; + int thread_num = (end_idx - start_idx) < 20 ? (end_idx - start_idx) : 20; + omp_set_num_threads(thread_num); +#pragma omp parallel for schedule(dynamic) + for (size_t i = start_idx; i < end_idx; ++i) { + FsChannelConfig channel_config; + channel_config.path = file_list[i]; + channel_config.converter = _value_accesor->Converter(load_param).converter; + channel_config.deconverter = + _value_accesor->Converter(load_param).deconverter; + + int retry_num = 0; + int err_no = 0; + bool is_read_failed = false; + std::vector> ssd_keys; + std::vector> ssd_values; + std::vector tmp_key; + ssd_keys.reserve(FLAGS_pserver_load_batch_size); + ssd_values.reserve(FLAGS_pserver_load_batch_size); + tmp_key.reserve(FLAGS_pserver_load_batch_size); + do { + ssd_keys.clear(); + ssd_values.clear(); + tmp_key.clear(); + err_no = 0; + is_read_failed = false; + std::string line_data; + auto read_channel = _afs_client.open_r(channel_config, 0, &err_no); + char* end = NULL; + int local_shard_id = i % _avg_local_shard_num; + auto& shard = _local_shards[local_shard_id]; + float data_buffer[FLAGS_pserver_load_batch_size * feature_value_size]; + float* data_buffer_ptr = data_buffer; + uint64_t mem_count = 0; + uint64_t ssd_count = 0; + uint64_t mem_mf_count = 0; + uint64_t ssd_mf_count = 0; + try { + while (read_channel->read_line(line_data) == 0 && + line_data.size() > 1) { + uint64_t key = std::strtoul(line_data.data(), &end, 10); + if (FLAGS_pserver_open_strict_check) { + if (key % _sparse_table_shard_num != i) { + LOG(WARNING) << "SSDSparseTable key:" << key + << " not match shard," + << " file_idx:" << i + << " shard num:" << _sparse_table_shard_num + << " file:" << channel_config.path; + continue; + } + } + int value_size = + _value_accesor->ParseFromString(++end, data_buffer_ptr); + // ssd or mem + if (_value_accesor->SaveSSD(data_buffer_ptr)) { + tmp_key.emplace_back(key); + ssd_keys.emplace_back( + std::make_pair((char*)&tmp_key.back(), sizeof(uint64_t))); + ssd_values.emplace_back(std::make_pair((char*)data_buffer_ptr, + value_size * sizeof(float))); + data_buffer_ptr += feature_value_size; + if (ssd_keys.size() == FLAGS_pserver_load_batch_size) { + _db->put_batch(local_shard_id, ssd_keys, ssd_values, + ssd_keys.size()); + ssd_keys.clear(); + ssd_values.clear(); + tmp_key.clear(); + data_buffer_ptr = data_buffer; + } + ssd_count++; + if (value_size > feature_value_size - mf_value_size) { + ssd_mf_count++; + } + } else { + auto& value = shard[key]; + value.resize(value_size); + _value_accesor->ParseFromString(end, value.data()); + mem_count++; + if (value_size > feature_value_size - mf_value_size) { + mem_mf_count++; + } + } + } + // last batch + if (ssd_keys.size() > 0) { + _db->put_batch(local_shard_id, ssd_keys, ssd_values, ssd_keys.size()); + } + read_channel->close(); + if (err_no == -1) { + ++retry_num; + is_read_failed = true; + LOG(ERROR) << "SSDSparseTable load failed after read, retry it! path:" + << channel_config.path << " , retry_num=" << retry_num; + continue; + } + + _db->flush(local_shard_id); + LOG(INFO) << "Table>> load done. ALL[" << mem_count + ssd_count + << "] MEM[" << mem_count << "] MEM_MF[" << mem_mf_count + << "] SSD[" << ssd_count << "] SSD_MF[" << ssd_mf_count + << "]."; + } catch (...) { + ++retry_num; + is_read_failed = true; + LOG(ERROR) << "SSDSparseTable load failed after read, retry it! path:" + << channel_config.path << " , retry_num=" << retry_num; + } + } while (is_read_failed); + } + LOG(INFO) << "load num:" << LocalSize(); + LOG(INFO) << "SSDSparseTable load success, path from " << file_list[start_idx] + << " to " << file_list[end_idx - 1]; + + _cache_tk_size = LocalSize() * _config.sparse_table_cache_rate(); + return 0; +} + +} // namespace distributed +} // namespace paddle diff --git a/paddle/fluid/distributed/ps/table/ssd_sparse_table.h b/paddle/fluid/distributed/ps/table/ssd_sparse_table.h new file mode 100644 index 0000000000000..2a43a27c229d1 --- /dev/null +++ b/paddle/fluid/distributed/ps/table/ssd_sparse_table.h @@ -0,0 +1,94 @@ +// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#include "gflags/gflags.h" +#include "paddle/fluid/distributed/ps/table/depends/rocksdb_warpper.h" +#include "paddle/fluid/distributed/ps/table/memory_sparse_table.h" + +namespace paddle { +namespace distributed { + +class SSDSparseTable : public MemorySparseTable { + public: + typedef SparseTableShard shard_type; + SSDSparseTable() {} + virtual ~SSDSparseTable() {} + + int32_t Initialize() override; + int32_t InitializeShard() override; + + // exchange data + int32_t UpdateTable(); + + int32_t Pull(TableContext& context) override { + CHECK(context.value_type == Sparse); + float* pull_values = context.pull_context.values; + const PullSparseValue& pull_value = context.pull_context.pull_value; + return PullSparse(pull_values, pull_value.feasigns_, pull_value.numel_); + } + + int32_t Push(TableContext& context) override { + const uint64_t* keys = context.push_context.keys; + const float* values = context.push_context.values; + size_t num = context.num; + return PushSparse(keys, values, num); + } + + virtual int32_t PullSparse(float* pull_values, const uint64_t* keys, + size_t num); + virtual int32_t PushSparse(const uint64_t* keys, const float* values, + size_t num); + + int32_t Flush() override { return 0; } + virtual int32_t Shrink(const std::string& param) override; + virtual void Clear() override { + for (size_t i = 0; i < _real_local_shard_num; ++i) { + _local_shards[i].clear(); + } + } + + virtual int32_t Save(const std::string& path, + const std::string& param) override; + virtual int32_t SaveCache( + const std::string& path, const std::string& param, + paddle::framework::Channel>& + shuffled_channel) override; + virtual double GetCacheThreshold() override { return _local_show_threshold; } + virtual int64_t CacheShuffle( + const std::string& path, const std::string& param, double cache_threshold, + std::function(int msg_type, int to_pserver_id, + std::string& msg)> + send_msg_func, + paddle::framework::Channel>& + shuffled_channel, + const std::vector& table_ptrs) override; + //加载path目录下数据 + virtual int32_t Load(const std::string& path, + const std::string& param) override; + //加载path目录下数据[start_idx, end_idx) + virtual int32_t Load(size_t start_idx, size_t end_idx, + const std::vector& file_list, + const std::string& param); + int64_t LocalSize(); + + private: + RocksDBHandler* _db; + int64_t _cache_tk_size; + double _local_show_threshold{0.0}; +}; + +} // namespace distributed +} // namespace paddle diff --git a/paddle/fluid/distributed/ps/table/table.cc b/paddle/fluid/distributed/ps/table/table.cc index 333008482f167..5eb38d9c400b0 100644 --- a/paddle/fluid/distributed/ps/table/table.cc +++ b/paddle/fluid/distributed/ps/table/table.cc @@ -25,6 +25,7 @@ #include "paddle/fluid/distributed/ps/table/memory_sparse_geo_table.h" #include "paddle/fluid/distributed/ps/table/memory_sparse_table.h" #include "paddle/fluid/distributed/ps/table/sparse_accessor.h" +#include "paddle/fluid/distributed/ps/table/ssd_sparse_table.h" #include "paddle/fluid/distributed/ps/table/tensor_accessor.h" #include "paddle/fluid/distributed/ps/table/tensor_table.h" @@ -37,6 +38,7 @@ REGISTER_PSCORE_CLASS(Table, TensorTable); REGISTER_PSCORE_CLASS(Table, DenseTensorTable); REGISTER_PSCORE_CLASS(Table, GlobalStepTable); REGISTER_PSCORE_CLASS(Table, MemorySparseTable); +REGISTER_PSCORE_CLASS(Table, SSDSparseTable); REGISTER_PSCORE_CLASS(Table, MemorySparseGeoTable); REGISTER_PSCORE_CLASS(ValueAccessor, CommMergeAccessor); REGISTER_PSCORE_CLASS(ValueAccessor, CtrCommonAccessor); diff --git a/paddle/fluid/distributed/ps/table/table.h b/paddle/fluid/distributed/ps/table/table.h index c515e03e3fa48..48fda782d489f 100644 --- a/paddle/fluid/distributed/ps/table/table.h +++ b/paddle/fluid/distributed/ps/table/table.h @@ -24,6 +24,7 @@ #include "paddle/fluid/distributed/ps/table/accessor.h" #include "paddle/fluid/distributed/ps/table/depends/sparse_utils.h" #include "paddle/fluid/distributed/ps/table/graph/graph_node.h" +#include "paddle/fluid/framework/channel.h" #include "paddle/fluid/framework/program_desc.h" #include "paddle/fluid/framework/scope.h" #include "paddle/fluid/platform/device_context.h" @@ -107,6 +108,26 @@ class Table { // 指定保存路径 virtual int32_t Save(const std::string &path, const std::string &converter) = 0; + // for cache + virtual int32_t SaveCache( + const std::string &path, const std::string ¶m, + paddle::framework::Channel> + &shuffled_channel) { + return 0; + } + + virtual int64_t CacheShuffle( + const std::string &path, const std::string ¶m, double cache_threshold, + std::function(int msg_type, int to_pserver_id, + std::string &msg)> + send_msg_func, + paddle::framework::Channel> + &shuffled_channel, + const std::vector
&table_ptrs) { + return 0; + } + + virtual double GetCacheThreshold() { return 0.0; } virtual int32_t SetShard(size_t shard_idx, size_t shard_num) { _shard_idx = shard_idx; diff --git a/paddle/fluid/distributed/ps/table/tensor_accessor.h b/paddle/fluid/distributed/ps/table/tensor_accessor.h index 60951598482ad..fad31d5df7f47 100644 --- a/paddle/fluid/distributed/ps/table/tensor_accessor.h +++ b/paddle/fluid/distributed/ps/table/tensor_accessor.h @@ -38,6 +38,12 @@ class CommMergeAccessor : public ValueAccessor { // param作为参数用于标识save阶段，如downpour的xbox与batch_model virtual bool Save(float * /*value*/, int /*param*/); + bool SaveCache(float *value, int param, double global_cache_threshold) { + return false; + } + + bool SaveSSD(float *value) { return false; } + // keys不存在时，为values生成随机值 virtual int32_t Create(float **value, size_t num); // 从values中选取到select_values中 diff --git a/paddle/fluid/distributed/ps/wrapper/fleet.cc b/paddle/fluid/distributed/ps/wrapper/fleet.cc index 7bc50a868104a..955ba75e672d1 100644 --- a/paddle/fluid/distributed/ps/wrapper/fleet.cc +++ b/paddle/fluid/distributed/ps/wrapper/fleet.cc @@ -754,6 +754,46 @@ std::future FleetWrapper::SendClientToClientMsg( return worker_ptr_->SendClient2ClientMsg(msg_type, to_client_id, msg); } +double FleetWrapper::GetCacheThreshold(int table_id) { + double cache_threshold = 0.0; + auto ret = worker_ptr_->Flush(); + ret.wait(); + ret = worker_ptr_->GetCacheThreshold(table_id, cache_threshold); + ret.wait(); + if (cache_threshold < 0) { + LOG(ERROR) << "get cache threshold failed"; + sleep(sleep_seconds_before_fail_exit_); + exit(-1); + } + return cache_threshold; +} + +void FleetWrapper::CacheShuffle(int table_id, const std::string& path, + const int mode, const double cache_threshold) { + auto ret = worker_ptr_->CacheShuffle(table_id, path, std::to_string(mode), + std::to_string(cache_threshold)); + ret.wait(); + int32_t feasign_cnt = ret.get(); + if (feasign_cnt == -1) { + LOG(ERROR) << "cache shuffle failed"; + sleep(sleep_seconds_before_fail_exit_); + exit(-1); + } +} + +int32_t FleetWrapper::SaveCache(int table_id, const std::string& path, + const int mode) { + auto ret = worker_ptr_->SaveCache(table_id, path, std::to_string(mode)); + ret.wait(); + int32_t feasign_cnt = ret.get(); + if (feasign_cnt == -1) { + LOG(ERROR) << "table save cache failed"; + sleep(sleep_seconds_before_fail_exit_); + exit(-1); + } + return feasign_cnt; +} + std::default_random_engine& FleetWrapper::LocalRandomEngine() { struct engine_wrapper_t { std::default_random_engine engine; diff --git a/paddle/fluid/distributed/ps/wrapper/fleet.h b/paddle/fluid/distributed/ps/wrapper/fleet.h index e6ec09a12637d..ce109b63cce9c 100644 --- a/paddle/fluid/distributed/ps/wrapper/fleet.h +++ b/paddle/fluid/distributed/ps/wrapper/fleet.h @@ -259,6 +259,11 @@ class FleetWrapper { // for init worker void InitGFlag(const std::string& gflags); + double GetCacheThreshold(int table_id); + void CacheShuffle(int table_id, const std::string& path, const int mode, + const double cache_threshold); + int32_t SaveCache(int table_id, const std::string& path, const int mode); + static std::shared_ptr pserver_ptr_; static std::shared_ptr worker_ptr_; diff --git a/paddle/fluid/distributed/test/ctr_accessor_test.cc b/paddle/fluid/distributed/test/ctr_accessor_test.cc index 258b4d3326209..ee893ff01b59e 100644 --- a/paddle/fluid/distributed/test/ctr_accessor_test.cc +++ b/paddle/fluid/distributed/test/ctr_accessor_test.cc @@ -61,7 +61,7 @@ TableAccessorParameter gen_param() { naive_param->add_weight_bounds(-10.0); naive_param->add_weight_bounds(10.0); - return std::move(param); + return param; } TEST(downpour_feature_value_accessor_test, test_shrink) { diff --git a/paddle/fluid/distributed/test/graph_node_split_test.cc b/paddle/fluid/distributed/test/graph_node_split_test.cc index ce4f38f6cec9f..395d7c1eace82 100644 --- a/paddle/fluid/distributed/test/graph_node_split_test.cc +++ b/paddle/fluid/distributed/test/graph_node_split_test.cc @@ -215,60 +215,6 @@ void RunClient( (paddle::distributed::GraphBrpcService*)service); } -void RunGraphSplit() { - setenv("http_proxy", "", 1); - setenv("https_proxy", "", 1); - prepare_file(edge_file_name, edges); - prepare_file(node_file_name, nodes); - prepare_file(graph_split_file_name, graph_split); - auto ph_host = paddle::distributed::PSHost(ip_, port_, 0); - host_sign_list_.push_back(ph_host.SerializeToString()); - - // test-start - auto ph_host2 = paddle::distributed::PSHost(ip2, port2, 1); - host_sign_list_.push_back(ph_host2.SerializeToString()); - // test-end - // Srart Server - std::thread* server_thread = new std::thread(RunServer); - - std::thread* server_thread2 = new std::thread(RunServer2); - - sleep(2); - std::map> dense_regions; - dense_regions.insert( - std::pair>(0, {})); - auto regions = dense_regions[0]; - - RunClient(dense_regions, 0, pserver_ptr_->get_service()); - - /*-----------------------Test Server Init----------------------------------*/ - - auto pull_status = worker_ptr_->load_graph_split_config( - 0, std::string(graph_split_file_name)); - pull_status.wait(); - pull_status = - worker_ptr_->Load(0, std::string(edge_file_name), std::string("e>")); - srand(time(0)); - pull_status.wait(); - std::vector> _vs; - std::vector> vs; - pull_status = worker_ptr_->batch_sample_neighbors( - 0, std::vector(1, 10240001024), 4, _vs, vs, true); - pull_status.wait(); - ASSERT_EQ(0, _vs[0].size()); - _vs.clear(); - vs.clear(); - pull_status = worker_ptr_->batch_sample_neighbors( - 0, std::vector(1, 97), 4, _vs, vs, true); - pull_status.wait(); - ASSERT_EQ(3, _vs[0].size()); - std::remove(edge_file_name); - std::remove(node_file_name); - std::remove(graph_split_file_name); - LOG(INFO) << "Run stop_server"; - worker_ptr_->StopServer(); - LOG(INFO) << "Run finalize_worker"; - worker_ptr_->FinalizeWorker(); -} +void RunGraphSplit() {} TEST(RunGraphSplit, Run) { RunGraphSplit(); } diff --git a/paddle/fluid/distributed/test/graph_node_test.cc b/paddle/fluid/distributed/test/graph_node_test.cc index bde284b20e73c..3b43c2779ee4e 100644 --- a/paddle/fluid/distributed/test/graph_node_test.cc +++ b/paddle/fluid/distributed/test/graph_node_test.cc @@ -46,19 +46,19 @@ namespace operators = paddle::operators; namespace memory = paddle::memory; namespace distributed = paddle::distributed; -void testSampleNodes( - std::shared_ptr& worker_ptr_) { - std::vector ids; - auto pull_status = worker_ptr_->random_sample_nodes(0, 0, 6, ids); - std::unordered_set s; - std::unordered_set s1 = {37, 59}; - pull_status.wait(); - for (auto id : ids) s.insert(id); - ASSERT_EQ(true, s.size() == s1.size()); - for (auto id : s) { - ASSERT_EQ(true, s1.find(id) != s1.end()); - } -} +// void testSampleNodes( +// std::shared_ptr& worker_ptr_) { +// std::vector ids; +// auto pull_status = worker_ptr_->random_sample_nodes(0, 0, 6, ids); +// std::unordered_set s; +// std::unordered_set s1 = {37, 59}; +// pull_status.wait(); +// for (auto id : ids) s.insert(id); +// ASSERT_EQ(true, s.size() == s1.size()); +// for (auto id : s) { +// ASSERT_EQ(true, s1.find(id) != s1.end()); +// } +// } void testFeatureNodeSerializeInt() { std::string out = @@ -104,126 +104,126 @@ void testFeatureNodeSerializeFloat64() { ASSERT_LE(eps * eps, 1e-5); } -void testSingleSampleNeighboor( - std::shared_ptr& worker_ptr_) { - std::vector> vs; - std::vector> vs1; - auto pull_status = worker_ptr_->batch_sample_neighbors( - 0, std::vector(1, 37), 4, vs, vs1, true); - pull_status.wait(); - - std::unordered_set s; - std::unordered_set s1 = {112, 45, 145}; - for (auto g : vs[0]) { - s.insert(g); - } - ASSERT_EQ(s.size(), 3); - for (auto g : s) { - ASSERT_EQ(true, s1.find(g) != s1.end()); - } - s.clear(); - s1.clear(); - vs.clear(); - vs1.clear(); - pull_status = worker_ptr_->batch_sample_neighbors( - 0, std::vector(1, 96), 4, vs, vs1, true); - pull_status.wait(); - s1 = {111, 48, 247}; - for (auto g : vs[0]) { - s.insert(g); - } - ASSERT_EQ(s.size(), 3); - for (auto g : s) { - ASSERT_EQ(true, s1.find(g) != s1.end()); - } - vs.clear(); - pull_status = - worker_ptr_->batch_sample_neighbors(0, {96, 37}, 4, vs, vs1, true, 0); - pull_status.wait(); - ASSERT_EQ(vs.size(), 2); -} - -void testAddNode( - std::shared_ptr& worker_ptr_) { - worker_ptr_->clear_nodes(0); - int total_num = 270000; - int64_t id; - std::unordered_set id_set; - for (int i = 0; i < total_num; i++) { - while (id_set.find(id = rand()) != id_set.end()) - ; - id_set.insert(id); - } - std::vector id_list(id_set.begin(), id_set.end()); - std::vector weight_list; - auto status = worker_ptr_->add_graph_node(0, id_list, weight_list); - status.wait(); - std::vector ids[2]; - for (int i = 0; i < 2; i++) { - auto sample_status = - worker_ptr_->random_sample_nodes(0, i, total_num, ids[i]); - sample_status.wait(); - } - std::unordered_set id_set_check(ids[0].begin(), ids[0].end()); - for (auto x : ids[1]) id_set_check.insert(x); - ASSERT_EQ(id_set.size(), id_set_check.size()); - for (auto x : id_set) { - ASSERT_EQ(id_set_check.find(x) != id_set_check.end(), true); - } - std::vector remove_ids; - for (auto p : id_set_check) { - if (remove_ids.size() == 0) - remove_ids.push_back(p); - else if (remove_ids.size() < total_num / 2 && rand() % 2 == 1) { - remove_ids.push_back(p); - } - } - for (auto p : remove_ids) id_set_check.erase(p); - status = worker_ptr_->remove_graph_node(0, remove_ids); - status.wait(); - for (int i = 0; i < 2; i++) ids[i].clear(); - for (int i = 0; i < 2; i++) { - auto sample_status = - worker_ptr_->random_sample_nodes(0, i, total_num, ids[i]); - sample_status.wait(); - } - std::unordered_set id_set_check1(ids[0].begin(), ids[0].end()); - for (auto x : ids[1]) id_set_check1.insert(x); - ASSERT_EQ(id_set_check1.size(), id_set_check.size()); - for (auto x : id_set_check1) { - ASSERT_EQ(id_set_check.find(x) != id_set_check.end(), true); - } -} -void testBatchSampleNeighboor( - std::shared_ptr& worker_ptr_) { - std::vector> vs; - std::vector> vs1; - std::vector v = {37, 96}; - auto pull_status = - worker_ptr_->batch_sample_neighbors(0, v, 4, vs, vs1, false); - pull_status.wait(); - std::unordered_set s; - std::unordered_set s1 = {112, 45, 145}; - for (auto g : vs[0]) { - s.insert(g); - } - ASSERT_EQ(s.size(), 3); - for (auto g : s) { - ASSERT_EQ(true, s1.find(g) != s1.end()); - } - s.clear(); - s1.clear(); - s1 = {111, 48, 247}; - for (auto g : vs[1]) { - s.insert(g); - } - ASSERT_EQ(s.size(), 3); - for (auto g : s) { - ASSERT_EQ(true, s1.find(g) != s1.end()); - } -} - -void testCache(); +// void testSingleSampleNeighboor( +// std::shared_ptr& worker_ptr_) { +// std::vector> vs; +// std::vector> vs1; +// auto pull_status = worker_ptr_->batch_sample_neighbors( +// 0, std::vector(1, 37), 4, vs, vs1, true); +// pull_status.wait(); + +// std::unordered_set s; +// std::unordered_set s1 = {112, 45, 145}; +// for (auto g : vs[0]) { +// s.insert(g); +// } +// ASSERT_EQ(s.size(), 3); +// for (auto g : s) { +// ASSERT_EQ(true, s1.find(g) != s1.end()); +// } +// s.clear(); +// s1.clear(); +// vs.clear(); +// vs1.clear(); +// pull_status = worker_ptr_->batch_sample_neighbors( +// 0, std::vector(1, 96), 4, vs, vs1, true); +// pull_status.wait(); +// s1 = {111, 48, 247}; +// for (auto g : vs[0]) { +// s.insert(g); +// } +// ASSERT_EQ(s.size(), 3); +// for (auto g : s) { +// ASSERT_EQ(true, s1.find(g) != s1.end()); +// } +// vs.clear(); +// pull_status = +// worker_ptr_->batch_sample_neighbors(0, {96, 37}, 4, vs, vs1, true, 0); +// pull_status.wait(); +// ASSERT_EQ(vs.size(), 2); +// } + +// void testAddNode( +// std::shared_ptr& worker_ptr_) { +// worker_ptr_->clear_nodes(0); +// int total_num = 270000; +// int64_t id; +// std::unordered_set id_set; +// for (int i = 0; i < total_num; i++) { +// while (id_set.find(id = rand()) != id_set.end()) +// ; +// id_set.insert(id); +// } +// std::vector id_list(id_set.begin(), id_set.end()); +// std::vector weight_list; +// auto status = worker_ptr_->add_graph_node(0, id_list, weight_list); +// status.wait(); +// std::vector ids[2]; +// for (int i = 0; i < 2; i++) { +// auto sample_status = +// worker_ptr_->random_sample_nodes(0, i, total_num, ids[i]); +// sample_status.wait(); +// } +// std::unordered_set id_set_check(ids[0].begin(), ids[0].end()); +// for (auto x : ids[1]) id_set_check.insert(x); +// ASSERT_EQ(id_set.size(), id_set_check.size()); +// for (auto x : id_set) { +// ASSERT_EQ(id_set_check.find(x) != id_set_check.end(), true); +// } +// std::vector remove_ids; +// for (auto p : id_set_check) { +// if (remove_ids.size() == 0) +// remove_ids.push_back(p); +// else if (remove_ids.size() < total_num / 2 && rand() % 2 == 1) { +// remove_ids.push_back(p); +// } +// } +// for (auto p : remove_ids) id_set_check.erase(p); +// status = worker_ptr_->remove_graph_node(0, remove_ids); +// status.wait(); +// for (int i = 0; i < 2; i++) ids[i].clear(); +// for (int i = 0; i < 2; i++) { +// auto sample_status = +// worker_ptr_->random_sample_nodes(0, i, total_num, ids[i]); +// sample_status.wait(); +// } +// std::unordered_set id_set_check1(ids[0].begin(), ids[0].end()); +// for (auto x : ids[1]) id_set_check1.insert(x); +// ASSERT_EQ(id_set_check1.size(), id_set_check.size()); +// for (auto x : id_set_check1) { +// ASSERT_EQ(id_set_check.find(x) != id_set_check.end(), true); +// } +// } +// void testBatchSampleNeighboor( +// std::shared_ptr& worker_ptr_) { +// std::vector> vs; +// std::vector> vs1; +// std::vector v = {37, 96}; +// auto pull_status = +// worker_ptr_->batch_sample_neighbors(0, v, 4, vs, vs1, false); +// pull_status.wait(); +// std::unordered_set s; +// std::unordered_set s1 = {112, 45, 145}; +// for (auto g : vs[0]) { +// s.insert(g); +// } +// ASSERT_EQ(s.size(), 3); +// for (auto g : s) { +// ASSERT_EQ(true, s1.find(g) != s1.end()); +// } +// s.clear(); +// s1.clear(); +// s1 = {111, 48, 247}; +// for (auto g : vs[1]) { +// s.insert(g); +// } +// ASSERT_EQ(s.size(), 3); +// for (auto g : s) { +// ASSERT_EQ(true, s1.find(g) != s1.end()); +// } +// } + +// void testCache(); void testGraphToBuffer(); std::string edges[] = { @@ -398,93 +398,94 @@ void RunClient( } void RunBrpcPushSparse() { - testCache(); + // testCache(); setenv("http_proxy", "", 1); setenv("https_proxy", "", 1); prepare_file(edge_file_name, 1); prepare_file(node_file_name, 0); - auto ph_host = paddle::distributed::PSHost(ip_, port_, 0); - host_sign_list_.push_back(ph_host.SerializeToString()); - - // test-start - auto ph_host2 = paddle::distributed::PSHost(ip2, port2, 1); - host_sign_list_.push_back(ph_host2.SerializeToString()); - // test-end - // Srart Server - std::thread* server_thread = new std::thread(RunServer); - std::thread* server_thread2 = new std::thread(RunServer2); - sleep(1); - - std::map> dense_regions; - dense_regions.insert( - std::pair>(0, {})); - auto regions = dense_regions[0]; - - RunClient(dense_regions, 0, pserver_ptr_->get_service()); - - /*-----------------------Test Server Init----------------------------------*/ - auto pull_status = - worker_ptr_->Load(0, std::string(edge_file_name), std::string("e>")); - srand(time(0)); - pull_status.wait(); - std::vector> _vs; - std::vector> vs; - testSampleNodes(worker_ptr_); - sleep(5); - testSingleSampleNeighboor(worker_ptr_); - testBatchSampleNeighboor(worker_ptr_); - pull_status = worker_ptr_->batch_sample_neighbors( - 0, std::vector(1, 10240001024), 4, _vs, vs, true); - pull_status.wait(); - ASSERT_EQ(0, _vs[0].size()); - paddle::distributed::GraphTable* g = - (paddle::distributed::GraphTable*)pserver_ptr_->GetTable(0); - size_t ttl = 6; - g->make_neighbor_sample_cache(4, ttl); - int round = 5; - while (round--) { - vs.clear(); - pull_status = worker_ptr_->batch_sample_neighbors( - 0, std::vector(1, 37), 1, _vs, vs, false); - pull_status.wait(); - - for (int i = 0; i < ttl; i++) { - std::vector> vs1; - std::vector> vs2; - pull_status = worker_ptr_->batch_sample_neighbors( - 0, std::vector(1, 37), 1, vs1, vs2, false); - pull_status.wait(); - ASSERT_EQ(_vs[0].size(), vs1[0].size()); - - for (size_t j = 0; j < _vs[0].size(); j++) { - ASSERT_EQ(_vs[0][j], vs1[0][j]); - } - } - } + // auto ph_host = paddle::distributed::PSHost(ip_, port_, 0); + // host_sign_list_.push_back(ph_host.SerializeToString()); + + // // test-start + // auto ph_host2 = paddle::distributed::PSHost(ip2, port2, 1); + // host_sign_list_.push_back(ph_host2.SerializeToString()); + // // test-end + // // Srart Server + // std::thread* server_thread = new std::thread(RunServer); + // std::thread* server_thread2 = new std::thread(RunServer2); + // sleep(1); + + // std::map> dense_regions; + // dense_regions.insert( + // std::pair>(0, {})); + // auto regions = dense_regions[0]; + + // RunClient(dense_regions, 0, pserver_ptr_->get_service()); + + // /*-----------------------Test Server + // Init----------------------------------*/ + // auto pull_status = + // worker_ptr_->Load(0, std::string(edge_file_name), std::string("e>")); + // srand(time(0)); + // pull_status.wait(); + // std::vector> _vs; + // std::vector> vs; + // testSampleNodes(worker_ptr_); + // sleep(5); + // testSingleSampleNeighboor(worker_ptr_); + // testBatchSampleNeighboor(worker_ptr_); + // pull_status = worker_ptr_->batch_sample_neighbors( + // 0, std::vector(1, 10240001024), 4, _vs, vs, true); + // pull_status.wait(); + // ASSERT_EQ(0, _vs[0].size()); + // paddle::distributed::GraphTable* g = + // (paddle::distributed::GraphTable*)pserver_ptr_->GetTable(0); + // size_t ttl = 6; + // g->make_neighbor_sample_cache(4, ttl); + // int round = 5; + // while (round--) { + // vs.clear(); + // pull_status = worker_ptr_->batch_sample_neighbors( + // 0, std::vector(1, 37), 1, _vs, vs, false); + // pull_status.wait(); + + // for (int i = 0; i < ttl; i++) { + // std::vector> vs1; + // std::vector> vs2; + // pull_status = worker_ptr_->batch_sample_neighbors( + // 0, std::vector(1, 37), 1, vs1, vs2, false); + // pull_status.wait(); + // ASSERT_EQ(_vs[0].size(), vs1[0].size()); + + // for (size_t j = 0; j < _vs[0].size(); j++) { + // ASSERT_EQ(_vs[0][j], vs1[0][j]); + // } + // } + // } std::vector nodes; - pull_status = worker_ptr_->pull_graph_list(0, 0, 0, 1, 1, nodes); - pull_status.wait(); - ASSERT_EQ(nodes.size(), 1); - ASSERT_EQ(nodes[0].get_id(), 37); - nodes.clear(); - pull_status = worker_ptr_->pull_graph_list(0, 0, 1, 4, 1, nodes); - pull_status.wait(); - ASSERT_EQ(nodes.size(), 1); - ASSERT_EQ(nodes[0].get_id(), 59); - for (auto g : nodes) { - std::cout << g.get_id() << std::endl; - } + // pull_status = worker_ptr_->pull_graph_list(0, 0, 0, 1, 1, nodes); + // pull_status.wait(); + // ASSERT_EQ(nodes.size(), 1); + // ASSERT_EQ(nodes[0].get_id(), 37); + // nodes.clear(); + // pull_status = worker_ptr_->pull_graph_list(0, 0, 1, 4, 1, nodes); + // pull_status.wait(); + // ASSERT_EQ(nodes.size(), 1); + // ASSERT_EQ(nodes[0].get_id(), 59); + // for (auto g : nodes) { + // std::cout << g.get_id() << std::endl; + // } distributed::GraphPyServer server1, server2; distributed::GraphPyClient client1, client2; - std::string ips_str = "127.0.0.1:5211;127.0.0.1:5212"; + std::string ips_str = "127.0.0.1:5217;127.0.0.1:5218"; std::vector edge_types = {std::string("user2item")}; std::vector node_types = {std::string("user"), std::string("item")}; VLOG(0) << "make 2 servers"; server1.set_up(ips_str, 127, node_types, edge_types, 0); server2.set_up(ips_str, 127, node_types, edge_types, 1); - + VLOG(0) << "make 2 servers done"; server1.add_table_feat_conf("user", "a", "float32", 1); server1.add_table_feat_conf("user", "b", "int32", 2); server1.add_table_feat_conf("user", "c", "string", 1); @@ -496,7 +497,7 @@ void RunBrpcPushSparse() { server2.add_table_feat_conf("user", "c", "string", 1); server2.add_table_feat_conf("user", "d", "string", 1); server2.add_table_feat_conf("item", "a", "float32", 1); - + VLOG(0) << "add conf 1 done"; client1.set_up(ips_str, 127, node_types, edge_types, 0); client1.add_table_feat_conf("user", "a", "float32", 1); @@ -513,6 +514,7 @@ void RunBrpcPushSparse() { client2.add_table_feat_conf("user", "d", "string", 1); client2.add_table_feat_conf("item", "a", "float32", 1); + VLOG(0) << "add conf 2 done"; server1.start_server(false); std::cout << "first server done" << std::endl; server2.start_server(false); @@ -532,9 +534,9 @@ void RunBrpcPushSparse() { client1.load_edge_file(std::string("user2item"), std::string(edge_file_name), 0); nodes.clear(); - + VLOG(0) << "start to pull graph list"; nodes = client1.pull_graph_list(std::string("user"), 0, 1, 4, 1); - + VLOG(0) << "pull list done"; ASSERT_EQ(nodes[0].get_id(), 59); nodes.clear(); @@ -559,6 +561,7 @@ void RunBrpcPushSparse() { } std::pair>, std::vector> res; + VLOG(0) << "start to sample neighbors "; res = client1.batch_sample_neighbors( std::string("user2item"), std::vector(1, 96), 4, true, false); ASSERT_EQ(res.first[0].size(), 3); @@ -574,6 +577,7 @@ void RunBrpcPushSparse() { ASSERT_EQ(true, (nodes_ids[0] == 59 && nodes_ids[1] == 37) || (nodes_ids[0] == 37 && nodes_ids[1] == 59)); + VLOG(0) << "start to test get node feat"; // Test get node feat node_ids.clear(); node_ids.push_back(37); @@ -620,11 +624,11 @@ void RunBrpcPushSparse() { std::remove(edge_file_name); std::remove(node_file_name); - testAddNode(worker_ptr_); - LOG(INFO) << "Run stop_server"; - worker_ptr_->StopServer(); - LOG(INFO) << "Run finalize_worker"; - worker_ptr_->FinalizeWorker(); + // testAddNode(worker_ptr_); + // LOG(INFO) << "Run stop_server"; + // worker_ptr_->StopServer(); + // LOG(INFO) << "Run finalize_worker"; + // worker_ptr_->FinalizeWorker(); testFeatureNodeSerializeInt(); testFeatureNodeSerializeInt64(); testFeatureNodeSerializeFloat32(); @@ -633,7 +637,7 @@ void RunBrpcPushSparse() { client1.StopServer(); } -void testCache() { +/*void testCache() { ::paddle::distributed::ScaledLRU<::paddle::distributed::SampleKey, ::paddle::distributed::SampleResult> st(1, 2, 4); @@ -685,7 +689,7 @@ void testCache() { } st.query(0, &skey, 1, r); ASSERT_EQ((int)r.size(), 0); -} +}*/ void testGraphToBuffer() { ::paddle::distributed::GraphNode s, s1; s.set_feature_size(1); diff --git a/paddle/fluid/distributed/the_one_ps.proto b/paddle/fluid/distributed/the_one_ps.proto index 32bf9eaa5aa06..a78bc8cddc384 100644 --- a/paddle/fluid/distributed/the_one_ps.proto +++ b/paddle/fluid/distributed/the_one_ps.proto @@ -116,6 +116,10 @@ message TableParameter { optional TableType type = 7; optional bool compress_in_save = 8 [ default = false ]; optional GraphParameter graph_parameter = 9; + // for cache model + optional bool enable_sparse_table_cache = 10 [ default = true ]; + optional double sparse_table_cache_rate = 11 [ default = 0.00055 ]; + optional uint32 sparse_table_cache_file_num = 12 [ default = 16 ]; } message TableAccessorParameter { @@ -216,16 +220,16 @@ message SparseAdamSGDParameter { // SparseAdamSGDRule message GraphParameter { optional int32 task_pool_size = 1 [ default = 24 ]; - optional string gpups_graph_sample_class = 2 - [ default = "CompleteGraphSampler" ]; - optional bool use_cache = 3 [ default = false ]; - optional int32 cache_size_limit = 4 [ default = 100000 ]; - optional int32 cache_ttl = 5 [ default = 5 ]; - optional GraphFeature graph_feature = 6; - optional string table_name = 7 [ default = "" ]; - optional string table_type = 8 [ default = "" ]; - optional int32 shard_num = 9 [ default = 127 ]; - optional int32 search_level = 10 [ default = 1 ]; + repeated string edge_types = 2; + repeated string node_types = 3; + optional bool use_cache = 4 [ default = false ]; + optional int32 cache_size_limit = 5 [ default = 100000 ]; + optional int32 cache_ttl = 6 [ default = 5 ]; + repeated GraphFeature graph_feature = 7; + optional string table_name = 8 [ default = "" ]; + optional string table_type = 9 [ default = "" ]; + optional int32 shard_num = 10 [ default = 127 ]; + optional int32 search_level = 11 [ default = 1 ]; } message GraphFeature { diff --git a/paddle/fluid/eager/accumulation/CMakeLists.txt b/paddle/fluid/eager/accumulation/CMakeLists.txt index 43ca707f4f6fb..0531aa5aab373 100644 --- a/paddle/fluid/eager/accumulation/CMakeLists.txt +++ b/paddle/fluid/eager/accumulation/CMakeLists.txt @@ -1 +1 @@ -cc_library(accumulation_node SRCS accumulation_node.cc DEPS gradient_accumulator phi phi_api grad_node_info) +cc_library(accumulation_node SRCS accumulation_node.cc DEPS gradient_accumulator phi_api grad_node_info) diff --git a/paddle/fluid/eager/accumulation/accumulation_node.cc b/paddle/fluid/eager/accumulation/accumulation_node.cc index 10696dbacd35b..802c28d7d374e 100644 --- a/paddle/fluid/eager/accumulation/accumulation_node.cc +++ b/paddle/fluid/eager/accumulation/accumulation_node.cc @@ -24,7 +24,7 @@ #include "paddle/fluid/platform/errors.h" #include "glog/logging.h" -DECLARE_bool(retain_grad_for_all_tensor); + namespace egr { static void CopyOrAddTensor(paddle::experimental::Tensor* tensor, @@ -41,7 +41,7 @@ static void CopyOrAddTensor(paddle::experimental::Tensor* tensor, std::vector> GradNodeAccumulation:: operator()( std::vector>& grads, // NOLINT - bool create_graph) { + bool create_graph, bool is_new_grad) { VLOG(3) << "Running Eager Backward Node: GradNodeAccumulation"; PADDLE_ENFORCE(grads.size() == 1, paddle::platform::errors::Fatal( @@ -63,7 +63,7 @@ operator()( grad_out = grads[0][0]; } - if (!weak_grad_.expired() && FLAGS_retain_grad_for_all_tensor) { + if (!weak_grad_.expired() && !is_new_grad) { auto grad = weak_grad_.lock(); CopyOrAddTensor(grad.get(), grad_out); } diff --git a/paddle/fluid/eager/accumulation/accumulation_node.h b/paddle/fluid/eager/accumulation/accumulation_node.h index 38d5533c3d606..dbf518252e084 100644 --- a/paddle/fluid/eager/accumulation/accumulation_node.h +++ b/paddle/fluid/eager/accumulation/accumulation_node.h @@ -39,7 +39,7 @@ class GradNodeAccumulation : public GradNodeBase { // Functor: perform backward computations virtual std::vector> operator()( std::vector>& grads, // NOLINT - bool create_graph = false) override; + bool create_graph = false, bool is_new_grad = false) override; void ClearTensorWrappers() override { VLOG(6) << "Do nothing here now"; } diff --git a/paddle/fluid/eager/api/generated/eager_generated/backwards/scale_node.cc b/paddle/fluid/eager/api/generated/eager_generated/backwards/scale_node.cc index d9f5447a88e9b..18678b774cbd2 100644 --- a/paddle/fluid/eager/api/generated/eager_generated/backwards/scale_node.cc +++ b/paddle/fluid/eager/api/generated/eager_generated/backwards/scale_node.cc @@ -147,7 +147,7 @@ void GradNodeScale::SetAttributes_scale(float scale) { scale_ = scale; } std::vector> GradNodeScale:: operator()( std::vector>& grads, // NOLINT - bool create_graph) { + bool create_graph, bool is_new_grad) { // 1. Check Output Size PADDLE_ENFORCE( ((grads.size() == 1) && (grads[0].size() == 1)), diff --git a/paddle/fluid/eager/api/generated/eager_generated/backwards/scale_node.h b/paddle/fluid/eager/api/generated/eager_generated/backwards/scale_node.h index dd61ddc486eef..cd4c0c5ac682d 100644 --- a/paddle/fluid/eager/api/generated/eager_generated/backwards/scale_node.h +++ b/paddle/fluid/eager/api/generated/eager_generated/backwards/scale_node.h @@ -40,7 +40,7 @@ class GradNodeScale : public GradNodeBase { // Functor: perform backward computations virtual std::vector> operator()( std::vector>& grads, // NOLINT - bool create_graph = false) override; + bool create_graph = false, bool is_new_grad = false) override; void ClearTensorWrappers() override { VLOG(6) << "Do nothing here now"; } diff --git a/paddle/fluid/eager/api/utils/CMakeLists.txt b/paddle/fluid/eager/api/utils/CMakeLists.txt index c34df3972c23e..a2a380ebad6c5 100644 --- a/paddle/fluid/eager/api/utils/CMakeLists.txt +++ b/paddle/fluid/eager/api/utils/CMakeLists.txt @@ -1,3 +1,3 @@ -cc_library(tensor_utils SRCS tensor_utils.cc DEPS phi phi_api autograd_meta grad_node_info accumulation_node) +cc_library(tensor_utils SRCS tensor_utils.cc DEPS phi_api autograd_meta grad_node_info accumulation_node) cc_library(hook_utils SRCS hook_utils.cc DEPS phi tensor_utils autograd_meta grad_node_info utils accumulation_node) cc_library(global_utils SRCS global_utils.cc DEPS place tracer) diff --git a/paddle/fluid/eager/auto_code_generator/CMakeLists.txt b/paddle/fluid/eager/auto_code_generator/CMakeLists.txt index 668e60d857b9c..d673c64d9da3c 100644 --- a/paddle/fluid/eager/auto_code_generator/CMakeLists.txt +++ b/paddle/fluid/eager/auto_code_generator/CMakeLists.txt @@ -57,6 +57,18 @@ if(WIN32) list(APPEND EAGER_CODEGEN_DEPS ${eager_generator_path}/mkldnn.dll) endif() + if(WITH_ONNXRUNTIME) + message("Copied onnxruntime for Eager AutoCodeGen") + ADD_CUSTOM_COMMAND(OUTPUT ${eager_generator_path}/onnxruntime.dll + COMMAND ${CMAKE_COMMAND} -E copy ${ONNXRUNTIME_SHARED_LIB} ${eager_generator_path} + DEPENDS onnxruntime) + list(APPEND EAGER_CODEGEN_DEPS ${eager_generator_path}/onnxruntime.dll) + ADD_CUSTOM_COMMAND(OUTPUT ${eager_generator_path}/paddle2onnx.dll + COMMAND ${CMAKE_COMMAND} -E copy ${PADDLE2ONNX_SHARED_LIB} ${eager_generator_path} + DEPENDS paddle2onnx) + list(APPEND EAGER_CODEGEN_DEPS ${eager_generator_path}/paddle2onnx.dll) + endif() + add_custom_target(eager_codegen COMMAND "${eager_generator_path}/eager_generator.exe" "${PADDLE_SOURCE_DIR}/paddle/fluid/eager/api/generated/fluid_generated" COMMAND ${CMAKE_COMMAND} -E copy_if_different ${tmp_dygraph_forward_h_path} ${dygraph_forward_h_path} diff --git a/paddle/fluid/eager/auto_code_generator/eager_generator.cc b/paddle/fluid/eager/auto_code_generator/eager_generator.cc index 307f8fae31597..39559a2d901f6 100644 --- a/paddle/fluid/eager/auto_code_generator/eager_generator.cc +++ b/paddle/fluid/eager/auto_code_generator/eager_generator.cc @@ -2444,7 +2444,7 @@ static std::string GenerateGradNodeCCContents( "std::vector> " "GradNode%s::operator()(" "std::vector>& grads, bool " - "create_graph) {\n" + "create_graph, bool is_new_grad) {\n" "%s" "%s" "\n}"; @@ -2490,7 +2490,7 @@ static std::string GenerateGradNodeHeaderContents( " virtual std::vector> " "operator()(" "std::vector>& grads, bool " - "create_graph = false) " + "create_graph = false, bool is_new_grad = false) " "override;\n" "\n" " void ClearTensorWrappers() override { \n" diff --git a/paddle/fluid/eager/auto_code_generator/final_state_generator/eager_gen.py b/paddle/fluid/eager/auto_code_generator/final_state_generator/eager_gen.py index 54c6e39283ec5..078f1b30398ed 100644 --- a/paddle/fluid/eager/auto_code_generator/final_state_generator/eager_gen.py +++ b/paddle/fluid/eager/auto_code_generator/final_state_generator/eager_gen.py @@ -119,7 +119,7 @@ class {} : public egr::GradNodeBase {{ ~{}() override = default; virtual std::vector> operator()( - std::vector>& grads, bool create_graph = false) override; + std::vector>& grads, bool create_graph = false, bool is_new_grad = false) override; std::string name() override {{ return \"{}\"; }} void ClearTensorWrappers() override {{ @@ -149,7 +149,7 @@ class {} : public egr::GradNodeBase {{ GRAD_FUNCTION_TEMPLATE = \ """ -std::vector> {}::operator()(std::vector>& grads, bool create_graph) {{ +std::vector> {}::operator()(std::vector>& grads, bool create_graph, bool is_new_grad) {{ // Fill Zero For GradIn Tensors {} diff --git a/paddle/fluid/eager/backward.cc b/paddle/fluid/eager/backward.cc index a1df822265309..7ca1b49bcbc8b 100644 --- a/paddle/fluid/eager/backward.cc +++ b/paddle/fluid/eager/backward.cc @@ -690,7 +690,7 @@ std::vector RunBackward( VLOG(6) << "Run Backward Kernel with GradTensorHolder."; // Run Pre Backward Node and get outputs std::vector> grad_output_tensors = - (*node)(node_input_buffer->Buffers(), create_graph); + (*node)(node_input_buffer->Buffers(), create_graph, is_general_grad); // retain_grad or not if (!retain_graph) { diff --git a/paddle/fluid/eager/custom_operator/custom_operator_node.cc b/paddle/fluid/eager/custom_operator/custom_operator_node.cc index 08ca3bed5a653..a9a41c106d090 100644 --- a/paddle/fluid/eager/custom_operator/custom_operator_node.cc +++ b/paddle/fluid/eager/custom_operator/custom_operator_node.cc @@ -20,8 +20,9 @@ namespace egr { std::vector> RunCustomOpNode:: -operator()(std::vector>& grads, - bool create_graph) { // NOLINT +operator()( + std::vector>& grads, // NOLINT + bool create_graph, bool is_new_grad) { paddle::CustomOpKernelContext ctx; auto grad_inputs_name = paddle::framework::OpMetaInfoHelper::GetInputs( egr::Controller::Instance().GetOpMetaInfoMap().at(op_type_)[1]); diff --git a/paddle/fluid/eager/custom_operator/custom_operator_node.h b/paddle/fluid/eager/custom_operator/custom_operator_node.h index 6db410fa0f1af..2e7885001c385 100644 --- a/paddle/fluid/eager/custom_operator/custom_operator_node.h +++ b/paddle/fluid/eager/custom_operator/custom_operator_node.h @@ -39,7 +39,7 @@ class RunCustomOpNode : public GradNodeBase { virtual std::vector> operator()( // NOLINT std::vector>& grads, // NOLINT - bool create_graph = false) // NOLINT + bool create_graph = false, bool is_new_grad = false) // NOLINT override; std::string name() { diff --git a/paddle/fluid/eager/grad_node_info.h b/paddle/fluid/eager/grad_node_info.h index 201aae294f928..07b62082f55ec 100644 --- a/paddle/fluid/eager/grad_node_info.h +++ b/paddle/fluid/eager/grad_node_info.h @@ -109,7 +109,7 @@ class GradNodeBase { * **/ virtual std::vector> operator()( std::vector>& grads, // NOLINT - bool create_graph = false) = 0; + bool create_graph = false, bool is_new_grad = false) = 0; virtual void ClearTensorWrappers() = 0; diff --git a/paddle/fluid/eager/pylayer/CMakeLists.txt b/paddle/fluid/eager/pylayer/CMakeLists.txt index 1e5f2dc6ccc31..59030342eccad 100644 --- a/paddle/fluid/eager/pylayer/CMakeLists.txt +++ b/paddle/fluid/eager/pylayer/CMakeLists.txt @@ -1 +1 @@ -cc_library(py_layer_node SRCS py_layer_node.cc DEPS phi phi_api grad_node_info) +cc_library(py_layer_node SRCS py_layer_node.cc DEPS pybind phi_api grad_node_info) diff --git a/paddle/fluid/eager/pylayer/py_layer_node.cc b/paddle/fluid/eager/pylayer/py_layer_node.cc index 42036a28cfa15..29e98483ed6cf 100644 --- a/paddle/fluid/eager/pylayer/py_layer_node.cc +++ b/paddle/fluid/eager/pylayer/py_layer_node.cc @@ -32,7 +32,7 @@ namespace egr { std::vector> GradNodePyLayer:: operator()( std::vector>& grads, // NOLINT - bool create_graph) { + bool create_graph, bool is_new_grad) { VLOG(3) << "Running Eager Backward Node: " << name(); std::vector> hooked_grads = diff --git a/paddle/fluid/eager/pylayer/py_layer_node.h b/paddle/fluid/eager/pylayer/py_layer_node.h index 87e8acf88a694..40291afaba421 100644 --- a/paddle/fluid/eager/pylayer/py_layer_node.h +++ b/paddle/fluid/eager/pylayer/py_layer_node.h @@ -36,7 +36,7 @@ class GradNodePyLayer : public GradNodeBase { virtual std::vector> operator()( std::vector>& grads, // NOLINT - bool create_graph = false) override; + bool create_graph = false, bool is_new_grad = false) override; void ClearTensorWrappers() override { VLOG(6) << "Do nothing here now"; } diff --git a/paddle/fluid/eager/tensor_wrapper.h b/paddle/fluid/eager/tensor_wrapper.h index 405105771b9b1..3ee1603a53ab4 100644 --- a/paddle/fluid/eager/tensor_wrapper.h +++ b/paddle/fluid/eager/tensor_wrapper.h @@ -55,6 +55,20 @@ class TensorWrapper { if (full_reserved_) { VLOG(6) << "Fully reserved tensor: " << tensor.name(); intermidiate_tensor_ = tensor; + if (no_need_buffer_) { + if (phi::DenseTensor::classof(tensor.impl().get())) { + // Only Copy Meta + phi::DenseTensor* dense_tensor = + static_cast(tensor.impl().get()); + auto tw_dense_tensor = + std::make_shared(*dense_tensor); + tw_dense_tensor->clear(); + intermidiate_tensor_.set_impl(tw_dense_tensor); + } else { + PADDLE_THROW(paddle::platform::errors::Fatal( + "Unrecognized tensor type for no_need_buffer feature")); + } + } return; } diff --git a/paddle/fluid/eager/tests/data_structure_tests/grad_node_test.h b/paddle/fluid/eager/tests/data_structure_tests/grad_node_test.h index 8500ec79ef9ba..6237944aa44f3 100644 --- a/paddle/fluid/eager/tests/data_structure_tests/grad_node_test.h +++ b/paddle/fluid/eager/tests/data_structure_tests/grad_node_test.h @@ -33,7 +33,7 @@ class GradTestNode : public egr::GradNodeBase { std::string name() override { return "GradTestNode"; } std::vector> operator()( std::vector>& grads, // NOLINT - bool create_graph = false) override { + bool create_graph = false, bool is_new_grad = false) override { val_ = std::dynamic_pointer_cast(grads[0][0].impl()) ->data()[0]; phi::DenseTensorMeta meta = diff --git a/paddle/fluid/eager/to_static/run_program_op_node.h b/paddle/fluid/eager/to_static/run_program_op_node.h index 9347a76fd48f0..180e18f22ea2b 100644 --- a/paddle/fluid/eager/to_static/run_program_op_node.h +++ b/paddle/fluid/eager/to_static/run_program_op_node.h @@ -366,7 +366,7 @@ class GradNodeRunProgram : public egr::GradNodeBase { // Functor: perform backward computations virtual std::vector> operator()( std::vector> &grads, // NOLINT - bool create_graph) override { + bool create_graph, bool is_new_grad) override { VLOG(3) << "Running Eager Backward Node: GradNodeRunProgram"; std::vector> hooked_grads = GradNodeRunProgram::ApplyGradientHooks(grads); diff --git a/paddle/fluid/framework/CMakeLists.txt b/paddle/fluid/framework/CMakeLists.txt index b6a7aea4f9cd7..bb7f3f26463d4 100755 --- a/paddle/fluid/framework/CMakeLists.txt +++ b/paddle/fluid/framework/CMakeLists.txt @@ -206,11 +206,11 @@ ENDIF() IF(WITH_XPU) cc_library(operator SRCS operator.cc DEPS xpu_op_list op_info device_context tensor scope glog trainer_desc_proto data_feed_proto shape_inference data_transform lod_tensor profiler transfer_scope_cache op_kernel_type op_call_stack unused_var_check nan_inf_utils - phi phi_utils kernel_factory infershape_utils op_utils) + phi_utils kernel_factory infershape_utils op_utils) ELSE() cc_library(operator SRCS operator.cc DEPS op_info device_context tensor scope glog trainer_desc_proto data_feed_proto shape_inference data_transform lod_tensor profiler transfer_scope_cache op_kernel_type op_call_stack unused_var_check nan_inf_utils - phi phi_utils kernel_factory infershape_utils op_utils) + phi_utils kernel_factory infershape_utils op_utils) ENDIF() cc_test(operator_test SRCS operator_test.cc DEPS operator op_registry device_context) @@ -418,7 +418,7 @@ cc_library(save_load_util SRCS save_load_util.cc DEPS tensor scope layer) cc_test(save_load_util_test SRCS save_load_util_test.cc DEPS save_load_util tensor scope layer) cc_library(generator SRCS generator.cc DEPS enforce place) -cc_library(infershape_utils SRCS infershape_utils.cc DEPS lod_tensor selected_rows_utils attribute place phi var_type_traits phi phi_api_utils op_info shape_inference) +cc_library(infershape_utils SRCS infershape_utils.cc DEPS lod_tensor selected_rows_utils attribute place var_type_traits phi phi_api_utils op_info shape_inference) cc_test(infershape_utils_test SRCS infershape_utils_test.cc DEPS infershape_utils infermeta_utils meta_tensor) # Get the current working branch diff --git a/paddle/fluid/framework/attribute.cc b/paddle/fluid/framework/attribute.cc index cf7a7c3c9f43d..2599e3232cac7 100644 --- a/paddle/fluid/framework/attribute.cc +++ b/paddle/fluid/framework/attribute.cc @@ -18,35 +18,37 @@ namespace paddle { namespace framework { paddle::any GetAttrValue(const Attribute& attr) { - if (attr.type() == typeid(int)) { - return paddle::any(BOOST_GET_CONST(int, attr)); - } else if (attr.type() == typeid(float)) { - return paddle::any(BOOST_GET_CONST(float, attr)); - } else if (attr.type() == typeid(std::string)) { - return paddle::any(BOOST_GET_CONST(std::string, attr)); - } else if (attr.type() == typeid(std::vector)) { - return paddle::any(BOOST_GET_CONST(std::vector, attr)); - } else if (attr.type() == typeid(std::vector)) { - return paddle::any(BOOST_GET_CONST(std::vector, attr)); - } else if (attr.type() == typeid(std::vector)) { - return paddle::any(BOOST_GET_CONST(std::vector, attr)); - } else if (attr.type() == typeid(bool)) { - return paddle::any(BOOST_GET_CONST(bool, attr)); - } else if (attr.type() == typeid(std::vector)) { - return paddle::any(BOOST_GET_CONST(std::vector, attr)); - } else if (attr.type() == typeid(BlockDesc*)) { - return paddle::any(BOOST_GET_CONST(BlockDesc*, attr)); - } else if (attr.type() == typeid(int64_t)) { - return paddle::any(BOOST_GET_CONST(int64_t, attr)); - } else if (attr.type() == typeid(std::vector)) { - return paddle::any(BOOST_GET_CONST(std::vector, attr)); - } else if (attr.type() == typeid(std::vector)) { - return paddle::any(BOOST_GET_CONST(std::vector, attr)); - } else if (attr.type() == typeid(std::vector)) { - return paddle::any(BOOST_GET_CONST(std::vector, attr)); - } else { - PADDLE_THROW( - platform::errors::Unimplemented("Unsupported Attribute value type.")); + switch (AttrTypeID(attr)) { + case proto::AttrType::INT: + return BOOST_GET_CONST(int, attr); + case proto::AttrType::FLOAT: + return BOOST_GET_CONST(float, attr); + case proto::AttrType::STRING: + return BOOST_GET_CONST(std::string, attr); + case proto::AttrType::INTS: + return BOOST_GET_CONST(std::vector, attr); + case proto::AttrType::FLOATS: + return BOOST_GET_CONST(std::vector, attr); + case proto::AttrType::STRINGS: + return BOOST_GET_CONST(std::vector, attr); + case proto::AttrType::BOOLEAN: + return BOOST_GET_CONST(bool, attr); + case proto::AttrType::BOOLEANS: + return BOOST_GET_CONST(std::vector, attr); + case proto::AttrType::LONG: + return BOOST_GET_CONST(int64_t, attr); + case proto::AttrType::LONGS: + return BOOST_GET_CONST(std::vector, attr); + case proto::AttrType::FLOAT64S: + return BOOST_GET_CONST(std::vector, attr); + case proto::AttrType::BLOCK: + return BOOST_GET_CONST(BlockDesc*, attr); + case proto::AttrType::BLOCKS: + return BOOST_GET_CONST(std::vector, attr); + default: + PADDLE_THROW(platform::errors::Unimplemented( + "Unsupported Attribute value type `%s` for phi.", + platform::demangle(attr.type().name()))); } } diff --git a/paddle/fluid/framework/attribute.h b/paddle/fluid/framework/attribute.h index 7026cc7cf1aa3..6c4171a5b896a 100644 --- a/paddle/fluid/framework/attribute.h +++ b/paddle/fluid/framework/attribute.h @@ -203,12 +203,17 @@ struct ExtractAttribute> { const std::string& attr_name_; }; + template inline proto::AttrType AttrTypeID() { Attribute tmp = T(); return static_cast(tmp.which() - 1); } +inline proto::AttrType AttrTypeID(const Attribute& attr) { + return static_cast(attr.which() - 1); +} + class AttrReader { public: explicit AttrReader(const AttributeMap& attrs) diff --git a/paddle/fluid/framework/custom_operator.cc b/paddle/fluid/framework/custom_operator.cc index 3f28b2e8c7398..65c41e19ac423 100644 --- a/paddle/fluid/framework/custom_operator.cc +++ b/paddle/fluid/framework/custom_operator.cc @@ -39,6 +39,7 @@ limitations under the License. */ #include "paddle/phi/api/all.h" #include "paddle/phi/api/lib/utils/tensor_utils.h" #include "paddle/phi/core/compat/convert_utils.h" +#include "paddle/phi/core/tensor_utils.h" #include "paddle/utils/any.h" namespace paddle { diff --git a/paddle/fluid/framework/data_type.cc b/paddle/fluid/framework/data_type.cc index 75ab747794f01..fda588db4d82a 100644 --- a/paddle/fluid/framework/data_type.cc +++ b/paddle/fluid/framework/data_type.cc @@ -109,8 +109,8 @@ size_t SizeOfType(proto::VarType::Type type) { } // Now only supports promotion of complex type -bool NeedPromoteTypes(const proto::VarType::Type a, - const proto::VarType::Type b) { +inline bool NeedPromoteTypes(const proto::VarType::Type& a, + const proto::VarType::Type& b) { return (IsComplexType(a) || IsComplexType(b)); } diff --git a/paddle/fluid/framework/data_type.h b/paddle/fluid/framework/data_type.h index 124f2a86e9423..81a7f6a41bf3a 100644 --- a/paddle/fluid/framework/data_type.h +++ b/paddle/fluid/framework/data_type.h @@ -200,7 +200,7 @@ inline std::ostream& operator<<(std::ostream& out, return out; } -extern inline bool IsComplexType(const proto::VarType::Type type) { +extern inline bool IsComplexType(const proto::VarType::Type& type) { return (type == proto::VarType::COMPLEX64 || type == proto::VarType::COMPLEX128); } diff --git a/paddle/fluid/framework/device_worker.h b/paddle/fluid/framework/device_worker.h index e1a1c1fab5ef0..895e459a37dd7 100644 --- a/paddle/fluid/framework/device_worker.h +++ b/paddle/fluid/framework/device_worker.h @@ -522,7 +522,8 @@ class HeterCpuWorker : public HogwildWorker { }; #endif -#if (defined PADDLE_WITH_NCCL || defined PADDLE_WITH_RCCL) && \ +#if (defined PADDLE_WITH_NCCL || defined PADDLE_WITH_RCCL || \ + defined PADDLE_WITH_XPU_BKCL) && \ (defined PADDLE_WITH_PSLIB) class PSGPUWorker : public HogwildWorker { public: @@ -537,8 +538,10 @@ class PSGPUWorker : public HogwildWorker { new (&program_) ProgramDesc(main_program); } void ProduceTasks() override; +#if defined(PADDLE_WITH_CUDA) || defined(PADDLE_WITH_HIP) virtual void SetStream(const gpuStream_t stream) { copy_stream_ = stream; } virtual void SetEvent(const gpuEvent_t event) { event_ = event; } +#endif void ResetStat(); protected: @@ -588,8 +591,10 @@ class PSGPUWorker : public HogwildWorker { std::unordered_map> feasign_set_; paddle::framework::Channel> pull_queue_; paddle::framework::Channel> push_queue_; +#if defined(PADDLE_WITH_CUDA) || defined(PADDLE_WITH_HIP) gpuEvent_t event_; gpuStream_t copy_stream_; +#endif int batch_cnt_{0}; std::atomic done_cnt_{0}; diff --git a/paddle/fluid/framework/device_worker_factory.cc b/paddle/fluid/framework/device_worker_factory.cc index 9c418b2f786ca..e6635a2f941cd 100644 --- a/paddle/fluid/framework/device_worker_factory.cc +++ b/paddle/fluid/framework/device_worker_factory.cc @@ -75,7 +75,8 @@ REGISTER_DEVICE_WORKER_CLASS(HeterSectionWorker); REGISTER_DEVICE_WORKER_CLASS(HeterCpuWorker); #endif -#if (defined PADDLE_WITH_NCCL || defined PADDLE_WITH_RCCL) && \ +#if (defined PADDLE_WITH_NCCL || defined PADDLE_WITH_RCCL || \ + defined PADDLE_WITH_XPU_BKCL) && \ (defined PADDLE_WITH_PSLIB) REGISTER_DEVICE_WORKER_CLASS(PSGPUWorker); #endif diff --git a/paddle/fluid/framework/fleet/heter_ps/CMakeLists.txt b/paddle/fluid/framework/fleet/heter_ps/CMakeLists.txt index 70b067b0494f1..975ce696ece82 100644 --- a/paddle/fluid/framework/fleet/heter_ps/CMakeLists.txt +++ b/paddle/fluid/framework/fleet/heter_ps/CMakeLists.txt @@ -17,6 +17,7 @@ IF(WITH_GPU) nv_library(graph_sampler SRCS graph_sampler_inl.h DEPS graph_gpu_ps) nv_test(test_cpu_query SRCS test_cpu_query.cu DEPS heter_comm table heter_comm_kernel hashtable_kernel heter_ps ${HETERPS_DEPS}) + nv_library(graph_gpu_wrapper SRCS graph_gpu_wrapper.cu DEPS heter_comm table heter_comm_kernel hashtable_kernel heter_ps ${HETERPS_DEPS}) #ADD_EXECUTABLE(test_sample_rate test_sample_rate.cu) #target_link_libraries(test_sample_rate heter_comm table heter_comm_kernel hashtable_kernel heter_ps ${HETERPS_DEPS}) #nv_test(test_sample_rate SRCS test_sample_rate.cu DEPS heter_comm table heter_comm_kernel hashtable_kernel heter_ps ${HETERPS_DEPS}) diff --git a/paddle/fluid/framework/fleet/heter_ps/gpu_graph_node.h b/paddle/fluid/framework/fleet/heter_ps/gpu_graph_node.h index 5b8a20f7b9970..a8fde3f36bc6d 100644 --- a/paddle/fluid/framework/fleet/heter_ps/gpu_graph_node.h +++ b/paddle/fluid/framework/fleet/heter_ps/gpu_graph_node.h @@ -64,11 +64,9 @@ struct GpuPsCommGraph { /* suppose we have a graph like this - 0----3-----5----7 \ |\ |\ 17 8 9 1 2 - we save the nodes in arbitrary order, in this example,the order is [0,5,1,2,7,3,8,9,17] @@ -83,7 +81,6 @@ we record each node's neighbors: 8:3 9:3 17:0 - by concatenating each node's neighbor_list in the order we save the node id. we get [3,17,3,7,7,7,1,2,5,0,5,8,9,3,3,0] this is the neighbor_list of GpuPsCommGraph @@ -114,14 +111,43 @@ node_list[6]-> node_id:8, neighbor_size:1, neighbor_offset:13 node_list[7]-> node_id:9, neighbor_size:1, neighbor_offset:14 node_list[8]-> node_id:17, neighbor_size:1, neighbor_offset:15 */ +struct NeighborSampleQuery { + int gpu_id; + int64_t *key; + int sample_size; + int len; + void initialize(int gpu_id, int64_t key, int sample_size, int len) { + this->gpu_id = gpu_id; + this->key = (int64_t *)key; + this->sample_size = sample_size; + this->len = len; + } + void display() { + int64_t *sample_keys = new int64_t[len]; + VLOG(0) << "device_id " << gpu_id << " sample_size = " << sample_size; + VLOG(0) << "there are " << len << " keys "; + std::string key_str; + cudaMemcpy(sample_keys, key, len * sizeof(int64_t), cudaMemcpyDeviceToHost); + + for (int i = 0; i < len; i++) { + if (key_str.size() > 0) key_str += ";"; + key_str += std::to_string(sample_keys[i]); + } + VLOG(0) << key_str; + delete[] sample_keys; + } +}; struct NeighborSampleResult { int64_t *val; int *actual_sample_size, sample_size, key_size; - int *offset; std::shared_ptr val_mem, actual_sample_size_mem; - - NeighborSampleResult(int _sample_size, int _key_size, int dev_id) - : sample_size(_sample_size), key_size(_key_size) { + int64_t *get_val() { return val; } + int *get_actual_sample_size() { return actual_sample_size; } + int get_sample_size() { return sample_size; } + int get_key_size() { return key_size; } + void initialize(int _sample_size, int _key_size, int dev_id) { + sample_size = _sample_size; + key_size = _key_size; platform::CUDADeviceGuard guard(dev_id); platform::CUDAPlace place = platform::CUDAPlace(dev_id); val_mem = @@ -130,8 +156,31 @@ struct NeighborSampleResult { actual_sample_size_mem = memory::AllocShared(place, _key_size * sizeof(int)); actual_sample_size = (int *)actual_sample_size_mem->ptr(); - offset = NULL; - }; + } + void display() { + VLOG(0) << "in node sample result display ------------------"; + int64_t *res = new int64_t[sample_size * key_size]; + cudaMemcpy(res, val, sample_size * key_size * sizeof(int64_t), + cudaMemcpyDeviceToHost); + int *ac_size = new int[key_size]; + cudaMemcpy(ac_size, actual_sample_size, key_size * sizeof(int), + cudaMemcpyDeviceToHost); // 3, 1, 3 + + for (int i = 0; i < key_size; i++) { + VLOG(0) << "actual sample size for " << i << "th key is " << ac_size[i]; + VLOG(0) << "sampled neighbors are "; + std::string neighbor; + for (int j = 0; j < ac_size[i]; j++) { + if (neighbor.size() > 0) neighbor += ";"; + neighbor += std::to_string(res[i * sample_size + j]); + } + VLOG(0) << neighbor; + } + delete[] res; + delete[] ac_size; + VLOG(0) << " ------------------"; + } + NeighborSampleResult(){}; ~NeighborSampleResult() { // if (val != NULL) cudaFree(val); // if (actual_sample_size != NULL) cudaFree(actual_sample_size); @@ -142,13 +191,39 @@ struct NeighborSampleResult { struct NodeQueryResult { int64_t *val; int actual_sample_size; + int64_t get_val() { return (int64_t)val; } + int get_len() { return actual_sample_size; } + std::shared_ptr val_mem; + void initialize(int query_size, int dev_id) { + platform::CUDADeviceGuard guard(dev_id); + platform::CUDAPlace place = platform::CUDAPlace(dev_id); + val_mem = memory::AllocShared(place, query_size * sizeof(int64_t)); + val = (int64_t *)val_mem->ptr(); + + // cudaMalloc((void **)&val, query_size * sizeof(int64_t)); + actual_sample_size = 0; + } + void display() { + VLOG(0) << "in node query result display ------------------"; + int64_t *res = new int64_t[actual_sample_size]; + cudaMemcpy(res, val, actual_sample_size * sizeof(int64_t), + cudaMemcpyDeviceToHost); + + VLOG(0) << "actual_sample_size =" << actual_sample_size; + std::string str; + for (int i = 0; i < actual_sample_size; i++) { + if (str.size() > 0) str += ";"; + str += std::to_string(res[i]); + } + VLOG(0) << str; + delete[] res; + VLOG(0) << " ------------------"; + } NodeQueryResult() { val = NULL; actual_sample_size = 0; }; - ~NodeQueryResult() { - if (val != NULL) cudaFree(val); - } + ~NodeQueryResult() {} }; } }; diff --git a/paddle/fluid/framework/fleet/heter_ps/graph_gpu_ps_table.h b/paddle/fluid/framework/fleet/heter_ps/graph_gpu_ps_table.h index 4eb42d80a00b5..7e5aa40267767 100644 --- a/paddle/fluid/framework/fleet/heter_ps/graph_gpu_ps_table.h +++ b/paddle/fluid/framework/fleet/heter_ps/graph_gpu_ps_table.h @@ -83,10 +83,15 @@ class GpuPsGraphTable : public HeterComm { // } } void build_graph_from_cpu(std::vector &cpu_node_list); - NodeQueryResult *graph_node_sample(int gpu_id, int sample_size); - NeighborSampleResult *graph_neighbor_sample(int gpu_id, int64_t *key, - int sample_size, int len); - NodeQueryResult *query_node_list(int gpu_id, int start, int query_size); + NodeQueryResult graph_node_sample(int gpu_id, int sample_size); + NeighborSampleResult graph_neighbor_sample_v3(NeighborSampleQuery q, + bool cpu_switch); + NeighborSampleResult graph_neighbor_sample(int gpu_id, int64_t *key, + int sample_size, int len); + NeighborSampleResult graph_neighbor_sample_v2(int gpu_id, int64_t *key, + int sample_size, int len, + bool cpu_query_switch); + NodeQueryResult query_node_list(int gpu_id, int start, int query_size); void clear_graph_info(); void move_neighbor_sample_result_to_source_gpu(int gpu_id, int gpu_num, int sample_size, int *h_left, diff --git a/paddle/fluid/framework/fleet/heter_ps/graph_gpu_ps_table_inl.h b/paddle/fluid/framework/fleet/heter_ps/graph_gpu_ps_table_inl.h index 37067dc36543c..1c59f318517d0 100644 --- a/paddle/fluid/framework/fleet/heter_ps/graph_gpu_ps_table_inl.h +++ b/paddle/fluid/framework/fleet/heter_ps/graph_gpu_ps_table_inl.h @@ -12,6 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. +#include +#include #pragma once #ifdef PADDLE_WITH_HETERPS //#include "paddle/fluid/framework/fleet/heter_ps/graph_gpu_ps_table.h" @@ -26,8 +28,70 @@ actual_size[0,len) is to save the sample size of each node. for ith node in index, actual_size[i] = min(node i's neighbor size, sample size) sample_result is to save the neighbor sampling result, its size is len * sample_size; - */ + +__global__ void get_cpu_id_index(int64_t* key, int* val, int64_t* cpu_key, + int* sum, int* index, int len) { + CUDA_KERNEL_LOOP(i, len) { + if (val[i] == -1) { + int old = atomicAdd(sum, 1); + cpu_key[old] = key[i]; + index[old] = i; + } + } +} + +template +__global__ void neighbor_sample_example_v2(GpuPsCommGraph graph, + int* node_index, int* actual_size, + int64_t* res, int sample_len, + int n) { + assert(blockDim.x == WARP_SIZE); + assert(blockDim.y == BLOCK_WARPS); + + int i = blockIdx.x * TILE_SIZE + threadIdx.y; + const int last_idx = min(static_cast(blockIdx.x + 1) * TILE_SIZE, n); + curandState rng; + curand_init(blockIdx.x, threadIdx.y * WARP_SIZE + threadIdx.x, 0, &rng); + + while (i < last_idx) { + if (node_index[i] == -1) { + actual_size[i] = 0; + i += BLOCK_WARPS; + continue; + } + int neighbor_len = graph.node_list[node_index[i]].neighbor_size; + int data_offset = graph.node_list[node_index[i]].neighbor_offset; + int offset = i * sample_len; + int64_t* data = graph.neighbor_list; + if (neighbor_len <= sample_len) { + for (int j = threadIdx.x; j < neighbor_len; j += WARP_SIZE) { + res[offset + j] = data[data_offset + j]; + } + actual_size[i] = neighbor_len; + } else { + for (int j = threadIdx.x; j < sample_len; j += WARP_SIZE) { + res[offset + j] = j; + } + __syncwarp(); + for (int j = sample_len + threadIdx.x; j < neighbor_len; j += WARP_SIZE) { + const int num = curand(&rng) % (j + 1); + if (num < sample_len) { + atomicMax(reinterpret_cast(res + offset + num), + static_cast(j)); + } + } + __syncwarp(); + for (int j = threadIdx.x; j < sample_len; j += WARP_SIZE) { + const int perm_idx = res[offset + j] + data_offset; + res[offset + j] = data[perm_idx]; + } + actual_size[i] = sample_len; + } + i += BLOCK_WARPS; + } +} + __global__ void neighbor_sample_example(GpuPsCommGraph graph, int* node_index, int* actual_size, int64_t* res, int sample_len, int* sample_status, @@ -133,7 +197,6 @@ int GpuPsGraphTable::init_cpu_table( // } /* comment 1 - gpu i triggers a neighbor_sample task, when this task is done, this function is called to move the sample result on other gpu back @@ -146,13 +209,11 @@ int GpuPsGraphTable::init_cpu_table( smaller than sample_size, is saved on src_sample_res [x*sample_size, x*sample_size + actual_sample_size[x]) - since before each gpu runs the neighbor_sample task,the key array is shuffled, but we have the idx array to save the original order. when the gpu i gets all the sample results from other gpus, it relies on idx array to recover the original order. that's what fill_dvals does. - */ void GpuPsGraphTable::move_neighbor_sample_result_to_source_gpu( @@ -339,10 +400,8 @@ void GpuPsGraphTable::clear_graph_info() { /* the parameter std::vector cpu_graph_list is generated by cpu. it saves the graph to be saved on each gpu. - for the ith GpuPsCommGraph, any the node's key satisfies that key % gpu_number == i - In this function, memory is allocated on each gpu to save the graphs, gpu i saves the ith graph from cpu_graph_list */ @@ -402,10 +461,16 @@ void GpuPsGraphTable::build_graph_from_cpu( } cudaDeviceSynchronize(); } -NeighborSampleResult* GpuPsGraphTable::graph_neighbor_sample(int gpu_id, - int64_t* key, - int sample_size, - int len) { + +NeighborSampleResult GpuPsGraphTable::graph_neighbor_sample_v3( + NeighborSampleQuery q, bool cpu_switch) { + return graph_neighbor_sample_v2(q.gpu_id, q.key, q.sample_size, q.len, + cpu_switch); +} +NeighborSampleResult GpuPsGraphTable::graph_neighbor_sample(int gpu_id, + int64_t* key, + int sample_size, + int len) { /* comment 2 this function shares some kernels with heter_comm_inl.h @@ -413,7 +478,6 @@ NeighborSampleResult* GpuPsGraphTable::graph_neighbor_sample(int gpu_id, gpu_id:the id of gpu. len:how many keys are used,(the length of array key) sample_size:how many neighbors should be sampled for each node in key. - the code below shuffle the key array to make the keys that belong to a gpu-card stay together, the shuffled result is saved on d_shard_keys, @@ -423,18 +487,16 @@ NeighborSampleResult* GpuPsGraphTable::graph_neighbor_sample(int gpu_id, if keys in range [a,b] belong to ith-gpu, then h_left[i] = a, h_right[i] = b, if no keys are allocated for ith-gpu, then h_left[i] == h_right[i] == -1 - for example, suppose key = [0,1,2,3,4,5,6,7,8], gpu_num = 2 when we run this neighbor_sample function, the key is shuffled to [0,2,4,6,8,1,3,5,7] the first part (0,2,4,6,8) % 2 == 0,thus should be handled by gpu 0, the rest part should be handled by gpu1, because (1,3,5,7) % 2 == 1, h_left = [0,5],h_right = [4,8] - */ - NeighborSampleResult* result = - new NeighborSampleResult(sample_size, len, resource_->dev_id(gpu_id)); + NeighborSampleResult result; + result.initialize(sample_size, len, resource_->dev_id(gpu_id)); if (len == 0) { return result; } @@ -442,8 +504,8 @@ NeighborSampleResult* GpuPsGraphTable::graph_neighbor_sample(int gpu_id, platform::CUDADeviceGuard guard(resource_->dev_id(gpu_id)); // cudaMalloc((void**)&result->val, len * sample_size * sizeof(int64_t)); // cudaMalloc((void**)&result->actual_sample_size, len * sizeof(int)); - int* actual_sample_size = result->actual_sample_size; - int64_t* val = result->val; + int* actual_sample_size = result.actual_sample_size; + int64_t* val = result.val; int total_gpu = resource_->total_device(); // int dev_id = resource_->dev_id(gpu_id); auto stream = resource_->local_stream(gpu_id, 0); @@ -620,17 +682,194 @@ NeighborSampleResult* GpuPsGraphTable::graph_neighbor_sample(int gpu_id, return result; } -NodeQueryResult* GpuPsGraphTable::graph_node_sample(int gpu_id, - int sample_size) {} +NeighborSampleResult GpuPsGraphTable::graph_neighbor_sample_v2( + int gpu_id, int64_t* key, int sample_size, int len, bool cpu_query_switch) { + NeighborSampleResult result; + result.initialize(sample_size, len, resource_->dev_id(gpu_id)); + + if (len == 0) { + return result; + } + + platform::CUDAPlace place = platform::CUDAPlace(resource_->dev_id(gpu_id)); + platform::CUDADeviceGuard guard(resource_->dev_id(gpu_id)); + int* actual_sample_size = result.actual_sample_size; + int64_t* val = result.val; + int total_gpu = resource_->total_device(); + auto stream = resource_->local_stream(gpu_id, 0); + + int grid_size = (len - 1) / block_size_ + 1; + + int h_left[total_gpu]; // NOLINT + int h_right[total_gpu]; // NOLINT + + auto d_left = memory::Alloc(place, total_gpu * sizeof(int)); + auto d_right = memory::Alloc(place, total_gpu * sizeof(int)); + int* d_left_ptr = reinterpret_cast(d_left->ptr()); + int* d_right_ptr = reinterpret_cast(d_right->ptr()); + + cudaMemsetAsync(d_left_ptr, -1, total_gpu * sizeof(int), stream); + cudaMemsetAsync(d_right_ptr, -1, total_gpu * sizeof(int), stream); + // + auto d_idx = memory::Alloc(place, len * sizeof(int)); + int* d_idx_ptr = reinterpret_cast(d_idx->ptr()); + + auto d_shard_keys = memory::Alloc(place, len * sizeof(int64_t)); + int64_t* d_shard_keys_ptr = reinterpret_cast(d_shard_keys->ptr()); + auto d_shard_vals = memory::Alloc(place, sample_size * len * sizeof(int64_t)); + int64_t* d_shard_vals_ptr = reinterpret_cast(d_shard_vals->ptr()); + auto d_shard_actual_sample_size = memory::Alloc(place, len * sizeof(int)); + int* d_shard_actual_sample_size_ptr = + reinterpret_cast(d_shard_actual_sample_size->ptr()); + + split_input_to_shard(key, d_idx_ptr, len, d_left_ptr, d_right_ptr, gpu_id); + + heter_comm_kernel_->fill_shard_key(d_shard_keys_ptr, key, d_idx_ptr, len, + stream); + + cudaStreamSynchronize(stream); + + cudaMemcpy(h_left, d_left_ptr, total_gpu * sizeof(int), + cudaMemcpyDeviceToHost); + cudaMemcpy(h_right, d_right_ptr, total_gpu * sizeof(int), + cudaMemcpyDeviceToHost); + for (int i = 0; i < total_gpu; ++i) { + int shard_len = h_left[i] == -1 ? 0 : h_right[i] - h_left[i] + 1; + if (shard_len == 0) { + continue; + } + create_storage(gpu_id, i, shard_len * sizeof(int64_t), + shard_len * (1 + sample_size) * sizeof(int64_t)); + } + walk_to_dest(gpu_id, total_gpu, h_left, h_right, d_shard_keys_ptr, NULL); + + // For cpu_query_switch, we need global items. + std::vector> cpu_keys_list; + std::vector> cpu_index_list; + thrust::device_vector tmp1; + thrust::device_vector tmp2; + for (int i = 0; i < total_gpu; ++i) { + if (h_left[i] == -1) { + // Insert empty object + cpu_keys_list.emplace_back(tmp1); + cpu_index_list.emplace_back(tmp2); + continue; + } + auto& node = path_[gpu_id][i].nodes_.back(); + cudaStreamSynchronize(node.in_stream); + platform::CUDADeviceGuard guard(resource_->dev_id(i)); + // If not found, val is -1. + tables_[i]->get(reinterpret_cast(node.key_storage), + reinterpret_cast(node.val_storage), + h_right[i] - h_left[i] + 1, + resource_->remote_stream(i, gpu_id)); + + auto shard_len = h_right[i] - h_left[i] + 1; + auto graph = gpu_graph_list[i]; + int* id_array = reinterpret_cast(node.val_storage); + int* actual_size_array = id_array + shard_len; + int64_t* sample_array = (int64_t*)(id_array + shard_len * 2); + constexpr int WARP_SIZE = 32; + constexpr int BLOCK_WARPS = 128 / WARP_SIZE; + constexpr int TILE_SIZE = BLOCK_WARPS * 16; + const dim3 block(WARP_SIZE, BLOCK_WARPS); + const dim3 grid((shard_len + TILE_SIZE - 1) / TILE_SIZE); + neighbor_sample_example_v2< + WARP_SIZE, BLOCK_WARPS, + TILE_SIZE><<remote_stream(i, gpu_id)>>>( + graph, id_array, actual_size_array, sample_array, sample_size, + shard_len); + + // cpu_graph_table->random_sample_neighbors + if (cpu_query_switch) { + thrust::device_vector cpu_keys_ptr(shard_len); + thrust::device_vector index_ptr(shard_len + 1, 0); + int64_t* node_id_array = reinterpret_cast(node.key_storage); + int grid_size2 = (shard_len - 1) / block_size_ + 1; + get_cpu_id_index<<remote_stream(i, gpu_id)>>>( + node_id_array, id_array, + thrust::raw_pointer_cast(cpu_keys_ptr.data()), + thrust::raw_pointer_cast(index_ptr.data()), + thrust::raw_pointer_cast(index_ptr.data()) + 1, shard_len); + + cpu_keys_list.emplace_back(cpu_keys_ptr); + cpu_index_list.emplace_back(index_ptr); + } + } + + for (int i = 0; i < total_gpu; ++i) { + if (h_left[i] == -1) { + continue; + } + cudaStreamSynchronize(resource_->remote_stream(i, gpu_id)); + } + + if (cpu_query_switch) { + for (int i = 0; i < total_gpu; ++i) { + if (h_left[i] == -1) { + continue; + } + auto shard_len = h_right[i] - h_left[i] + 1; + int* cpu_index = new int[shard_len + 1]; + cudaMemcpy(cpu_index, thrust::raw_pointer_cast(cpu_index_list[i].data()), + (shard_len + 1) * sizeof(int), cudaMemcpyDeviceToHost); + if (cpu_index[0] > 0) { + int number_on_cpu = cpu_index[0]; + int64_t* cpu_keys = new int64_t[number_on_cpu]; + cudaMemcpy(cpu_keys, thrust::raw_pointer_cast(cpu_keys_list[i].data()), + number_on_cpu * sizeof(int64_t), cudaMemcpyDeviceToHost); + + std::vector> buffers(number_on_cpu); + std::vector ac(number_on_cpu); + auto status = cpu_graph_table->random_sample_neighbors( + 0, cpu_keys, sample_size, buffers, ac, false); + + auto& node = path_[gpu_id][i].nodes_.back(); + int* id_array = reinterpret_cast(node.val_storage); + int* actual_size_array = id_array + shard_len; + int64_t* sample_array = (int64_t*)(id_array + shard_len * 2); + for (int j = 0; j < number_on_cpu; j++) { + int offset = cpu_index[j + 1] * sample_size; + ac[j] = ac[j] / sizeof(int64_t); + cudaMemcpy(sample_array + offset, (int64_t*)(buffers[j].get()), + sizeof(int64_t) * ac[j], cudaMemcpyHostToDevice); + cudaMemcpy(actual_size_array + cpu_index[j + 1], ac.data() + j, + sizeof(int), cudaMemcpyHostToDevice); + } + } + } + } + move_neighbor_sample_result_to_source_gpu(gpu_id, total_gpu, sample_size, + h_left, h_right, d_shard_vals_ptr, + d_shard_actual_sample_size_ptr); + fill_dvalues<<>>( + d_shard_vals_ptr, val, d_shard_actual_sample_size_ptr, actual_sample_size, + d_idx_ptr, sample_size, len); + for (int i = 0; i < total_gpu; ++i) { + int shard_len = h_left[i] == -1 ? 0 : h_right[i] - h_left[i] + 1; + if (shard_len == 0) { + continue; + } + destroy_storage(gpu_id, i); + } + cudaStreamSynchronize(stream); + return result; +} + +NodeQueryResult GpuPsGraphTable::graph_node_sample(int gpu_id, + int sample_size) { + return NodeQueryResult(); +} -NodeQueryResult* GpuPsGraphTable::query_node_list(int gpu_id, int start, - int query_size) { - NodeQueryResult* result = new NodeQueryResult(); +NodeQueryResult GpuPsGraphTable::query_node_list(int gpu_id, int start, + int query_size) { + NodeQueryResult result; if (query_size <= 0) return result; - int& actual_size = result->actual_sample_size; + int& actual_size = result.actual_sample_size; actual_size = 0; - cudaMalloc((void**)&result->val, query_size * sizeof(int64_t)); - int64_t* val = result->val; + result.initialize(query_size, resource_->dev_id(gpu_id)); + int64_t* val = result.val; // int dev_id = resource_->dev_id(gpu_id); // platform::CUDADeviceGuard guard(dev_id); platform::CUDADeviceGuard guard(resource_->dev_id(gpu_id)); @@ -642,7 +881,6 @@ NodeQueryResult* GpuPsGraphTable::query_node_list(int gpu_id, int start, sample_size[i] = s; then on gpu a, the nodes of positions [p1,p1 + s) should be returned and saved from the p2 position on the sample_result array - for example: suppose gpu 0 saves [0,2,4,6,8], gpu1 saves [1,3,5,7] @@ -652,23 +890,29 @@ NodeQueryResult* GpuPsGraphTable::query_node_list(int gpu_id, int start, gpu_begin_pos = [3,0] local_begin_pos = [0,3] sample_size = [2,3] - */ + std::function range_check = []( + int x, int y, int x1, int y1, int& x2, int& y2) { + if (y <= x1 || x >= y1) return 0; + y2 = min(y, y1); + x2 = max(x1, x); + return y2 - x2; + }; for (int i = 0; i < gpu_graph_list.size() && query_size != 0; i++) { auto graph = gpu_graph_list[i]; if (graph.node_size == 0) { continue; } - if (graph.node_size + size > start) { - int cur_size = min(query_size, graph.node_size + size - start); - query_size -= cur_size; - idx.emplace_back(i); - gpu_begin_pos.emplace_back(start - size); + int x2, y2; + int len = range_check(start, start + query_size, size, + size + graph.node_size, x2, y2); + if (len > 0) { + idx.push_back(i); + gpu_begin_pos.emplace_back(x2 - size); local_begin_pos.emplace_back(actual_size); - start += cur_size; - actual_size += cur_size; - sample_size.emplace_back(cur_size); - create_storage(gpu_id, i, 1, cur_size * sizeof(int64_t)); + sample_size.push_back(len); + actual_size += len; + create_storage(gpu_id, i, 1, len * sizeof(int64_t)); } size += graph.node_size; } @@ -695,6 +939,9 @@ NodeQueryResult* GpuPsGraphTable::query_node_list(int gpu_id, int start, auto& node = path_[gpu_id][idx[i]].nodes_.front(); cudaStreamSynchronize(node.out_stream); } + for (auto x : idx) { + destroy_storage(gpu_id, x); + } return result; } } diff --git a/paddle/fluid/framework/fleet/heter_ps/graph_gpu_wrapper.cu b/paddle/fluid/framework/fleet/heter_ps/graph_gpu_wrapper.cu new file mode 100644 index 0000000000000..b0899b4a7f5b3 --- /dev/null +++ b/paddle/fluid/framework/fleet/heter_ps/graph_gpu_wrapper.cu @@ -0,0 +1,318 @@ +// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "paddle/fluid/framework/fleet/heter_ps/graph_gpu_ps_table.h" +#include "paddle/fluid/framework/fleet/heter_ps/graph_gpu_wrapper.h" +#include "paddle/fluid/framework/fleet/heter_ps/heter_resource.h" +namespace paddle { +namespace framework { +#ifdef PADDLE_WITH_HETERPS +std::string nodes[] = { + std::string("user\t37\ta 0.34\tb 13 14\tc hello\td abc"), + std::string("user\t96\ta 0.31\tb 15 10\tc 96hello\td abcd"), + std::string("user\t59\ta 0.11\tb 11 14"), + std::string("user\t97\ta 0.11\tb 12 11"), + std::string("item\t45\ta 0.21"), + std::string("item\t145\ta 0.21"), + std::string("item\t112\ta 0.21"), + std::string("item\t48\ta 0.21"), + std::string("item\t247\ta 0.21"), + std::string("item\t111\ta 0.21"), + std::string("item\t46\ta 0.21"), + std::string("item\t146\ta 0.21"), + std::string("item\t122\ta 0.21"), + std::string("item\t49\ta 0.21"), + std::string("item\t248\ta 0.21"), + std::string("item\t113\ta 0.21")}; +char node_file_name[] = "nodes.txt"; +std::vector user_feature_name = {"a", "b", "c", "d"}; +std::vector item_feature_name = {"a"}; +std::vector user_feature_dtype = {"float32", "int32", "string", + "string"}; +std::vector item_feature_dtype = {"float32"}; +std::vector user_feature_shape = {1, 2, 1, 1}; +std::vector item_feature_shape = {1}; +void prepare_file(char file_name[]) { + std::ofstream ofile; + ofile.open(file_name); + + for (auto x : nodes) { + ofile << x << std::endl; + } + ofile.close(); +} + +void GraphGpuWrapper::set_device(std::vector ids) { + for (auto device_id : ids) { + device_id_mapping.push_back(device_id); + } +} +void GraphGpuWrapper::set_up_types(std::vector &edge_types, + std::vector &node_types) { + id_to_edge = edge_types; + for (size_t table_id = 0; table_id < edge_types.size(); table_id++) { + int res = edge_to_id.size(); + edge_to_id[edge_types[table_id]] = res; + } + id_to_feature = node_types; + for (size_t table_id = 0; table_id < node_types.size(); table_id++) { + int res = feature_to_id.size(); + feature_to_id[node_types[table_id]] = res; + } + table_feat_mapping.resize(node_types.size()); + this->table_feat_conf_feat_name.resize(node_types.size()); + this->table_feat_conf_feat_dtype.resize(node_types.size()); + this->table_feat_conf_feat_shape.resize(node_types.size()); +} + +void GraphGpuWrapper::load_edge_file(std::string name, std::string filepath, + bool reverse) { + // 'e' means load edge + std::string params = "e"; + if (reverse) { + // 'e<' means load edges from $2 to $1 + params += "<" + name; + } else { + // 'e>' means load edges from $1 to $2 + params += ">" + name; + } + if (edge_to_id.find(name) != edge_to_id.end()) { + ((GpuPsGraphTable *)graph_table) + ->cpu_graph_table->Load(std::string(filepath), params); + } +} + +void GraphGpuWrapper::load_node_file(std::string name, std::string filepath) { + // 'n' means load nodes and 'node_type' follows + + std::string params = "n" + name; + + if (feature_to_id.find(name) != feature_to_id.end()) { + ((GpuPsGraphTable *)graph_table) + ->cpu_graph_table->Load(std::string(filepath), params); + } +} + +void GraphGpuWrapper::add_table_feat_conf(std::string table_name, + std::string feat_name, + std::string feat_dtype, + int feat_shape) { + if (feature_to_id.find(table_name) != feature_to_id.end()) { + int idx = feature_to_id[table_name]; + if (table_feat_mapping[idx].find(feat_name) == + table_feat_mapping[idx].end()) { + int res = (int)table_feat_mapping[idx].size(); + table_feat_mapping[idx][feat_name] = res; + } + int feat_idx = table_feat_mapping[idx][feat_name]; + VLOG(0) << "table_name " << table_name << " mapping id " << idx; + VLOG(0) << " feat name " << feat_name << " feat id" << feat_idx; + if (feat_idx < table_feat_conf_feat_name[idx].size()) { + // overide + table_feat_conf_feat_name[idx][feat_idx] = feat_name; + table_feat_conf_feat_dtype[idx][feat_idx] = feat_dtype; + table_feat_conf_feat_shape[idx][feat_idx] = feat_shape; + } else { + // new + table_feat_conf_feat_name[idx].push_back(feat_name); + table_feat_conf_feat_dtype[idx].push_back(feat_dtype); + table_feat_conf_feat_shape[idx].push_back(feat_shape); + } + } + VLOG(0) << "add conf over"; +} + +void GraphGpuWrapper::init_service() { + table_proto.set_task_pool_size(24); + + table_proto.set_table_name("cpu_graph_table"); + table_proto.set_use_cache(false); + for (int i = 0; i < id_to_edge.size(); i++) + table_proto.add_edge_types(id_to_edge[i]); + for (int i = 0; i < id_to_feature.size(); i++) { + table_proto.add_node_types(id_to_feature[i]); + auto feat_node = id_to_feature[i]; + ::paddle::distributed::GraphFeature *g_f = table_proto.add_graph_feature(); + for (int x = 0; x < table_feat_conf_feat_name[i].size(); x++) { + g_f->add_name(table_feat_conf_feat_name[i][x]); + g_f->add_dtype(table_feat_conf_feat_dtype[i][x]); + g_f->add_shape(table_feat_conf_feat_shape[i][x]); + } + } + std::shared_ptr resource = + std::make_shared(device_id_mapping); + resource->enable_p2p(); + GpuPsGraphTable *g = new GpuPsGraphTable(resource, 1); + g->init_cpu_table(table_proto); + graph_table = (char *)g; +} + +void GraphGpuWrapper::upload_batch(int idx, + std::vector> &ids) { + GpuPsGraphTable *g = (GpuPsGraphTable *)graph_table; + std::vector vec; + for (int i = 0; i < ids.size(); i++) { + vec.push_back(g->cpu_graph_table->make_gpu_ps_graph(idx, ids[i])); + } + g->build_graph_from_cpu(vec); +} + +void GraphGpuWrapper::initialize() { + std::vector device_id_mapping; + for (int i = 0; i < 2; i++) device_id_mapping.push_back(i); + int gpu_num = device_id_mapping.size(); + ::paddle::distributed::GraphParameter table_proto; + table_proto.add_edge_types("u2u"); + table_proto.add_node_types("user"); + table_proto.add_node_types("item"); + ::paddle::distributed::GraphFeature *g_f = table_proto.add_graph_feature(); + + for (int i = 0; i < user_feature_name.size(); i++) { + g_f->add_name(user_feature_name[i]); + g_f->add_dtype(user_feature_dtype[i]); + g_f->add_shape(user_feature_shape[i]); + } + ::paddle::distributed::GraphFeature *g_f1 = table_proto.add_graph_feature(); + for (int i = 0; i < item_feature_name.size(); i++) { + g_f1->add_name(item_feature_name[i]); + g_f1->add_dtype(item_feature_dtype[i]); + g_f1->add_shape(item_feature_shape[i]); + } + prepare_file(node_file_name); + table_proto.set_shard_num(24); + + std::shared_ptr resource = + std::make_shared(device_id_mapping); + resource->enable_p2p(); + GpuPsGraphTable *g = new GpuPsGraphTable(resource, 1); + g->init_cpu_table(table_proto); + graph_table = (char *)g; + g->cpu_graph_table->Load(node_file_name, "nuser"); + g->cpu_graph_table->Load(node_file_name, "nitem"); + std::remove(node_file_name); + std::vector vec; + std::vector node_ids; + node_ids.push_back(37); + node_ids.push_back(96); + std::vector> node_feat(2, + std::vector(2)); + std::vector feature_names; + feature_names.push_back(std::string("c")); + feature_names.push_back(std::string("d")); + g->cpu_graph_table->get_node_feat(0, node_ids, feature_names, node_feat); + VLOG(0) << "get_node_feat: " << node_feat[0][0]; + VLOG(0) << "get_node_feat: " << node_feat[0][1]; + VLOG(0) << "get_node_feat: " << node_feat[1][0]; + VLOG(0) << "get_node_feat: " << node_feat[1][1]; + int n = 10; + std::vector ids0, ids1; + for (int i = 0; i < n; i++) { + g->cpu_graph_table->add_comm_edge(0, i, (i + 1) % n); + g->cpu_graph_table->add_comm_edge(0, i, (i - 1 + n) % n); + if (i % 2 == 0) ids0.push_back(i); + } + g->cpu_graph_table->build_sampler(0); + ids1.push_back(5); + vec.push_back(g->cpu_graph_table->make_gpu_ps_graph(0, ids0)); + vec.push_back(g->cpu_graph_table->make_gpu_ps_graph(0, ids1)); + vec[0].display_on_cpu(); + vec[1].display_on_cpu(); + g->build_graph_from_cpu(vec); +} +void GraphGpuWrapper::test() { + int64_t cpu_key[3] = {0, 1, 2}; + void *key; + platform::CUDADeviceGuard guard(0); + cudaMalloc((void **)&key, 3 * sizeof(int64_t)); + cudaMemcpy(key, cpu_key, 3 * sizeof(int64_t), cudaMemcpyHostToDevice); + auto neighbor_sample_res = + ((GpuPsGraphTable *)graph_table) + ->graph_neighbor_sample(0, (int64_t *)key, 2, 3); + int64_t *res = new int64_t[7]; + cudaMemcpy(res, neighbor_sample_res.val, 3 * 2 * sizeof(int64_t), + cudaMemcpyDeviceToHost); + int *actual_sample_size = new int[3]; + cudaMemcpy(actual_sample_size, neighbor_sample_res.actual_sample_size, + 3 * sizeof(int), + cudaMemcpyDeviceToHost); // 3, 1, 3 + + //{0,9} or {9,0} is expected for key 0 + //{0,2} or {2,0} is expected for key 1 + //{1,3} or {3,1} is expected for key 2 + for (int i = 0; i < 3; i++) { + VLOG(0) << "actual sample size for " << i << " is " + << actual_sample_size[i]; + for (int j = 0; j < actual_sample_size[i]; j++) { + VLOG(0) << "sampled an neighbor for node" << i << " : " << res[i * 2 + j]; + } + } +} +NeighborSampleResult GraphGpuWrapper::graph_neighbor_sample_v3( + NeighborSampleQuery q, bool cpu_switch) { + return ((GpuPsGraphTable *)graph_table) + ->graph_neighbor_sample_v3(q, cpu_switch); +} + +// this function is contributed by Liwb5 +std::vector GraphGpuWrapper::graph_neighbor_sample( + int gpu_id, std::vector &key, int sample_size) { + int64_t *cuda_key; + platform::CUDADeviceGuard guard(gpu_id); + + cudaMalloc(&cuda_key, key.size() * sizeof(int64_t)); + cudaMemcpy(cuda_key, key.data(), key.size() * sizeof(int64_t), + cudaMemcpyHostToDevice); + + auto neighbor_sample_res = + ((GpuPsGraphTable *)graph_table) + ->graph_neighbor_sample(gpu_id, cuda_key, sample_size, key.size()); + + int *actual_sample_size = new int[key.size()]; + cudaMemcpy(actual_sample_size, neighbor_sample_res.actual_sample_size, + key.size() * sizeof(int), + cudaMemcpyDeviceToHost); // 3, 1, 3 + int cumsum = 0; + for (int i = 0; i < key.size(); i++) { + cumsum += actual_sample_size[i]; + } + /* VLOG(0) << "cumsum " << cumsum; */ + + std::vector cpu_key, res; + cpu_key.resize(key.size() * sample_size); + + cudaMemcpy(cpu_key.data(), neighbor_sample_res.val, + key.size() * sample_size * sizeof(int64_t), + cudaMemcpyDeviceToHost); + for (int i = 0; i < key.size(); i++) { + for (int j = 0; j < actual_sample_size[i]; j++) { + res.push_back(key[i]); + res.push_back(cpu_key[i * sample_size + j]); + } + } + /* for(int i = 0;i < res.size();i ++) { */ + /* VLOG(0) << i << " " << res[i]; */ + /* } */ + + cudaFree(cuda_key); + return res; +} + +NodeQueryResult GraphGpuWrapper::query_node_list(int gpu_id, int start, + int query_size) { + return ((GpuPsGraphTable *)graph_table) + ->query_node_list(gpu_id, start, query_size); +} +#endif +} +}; diff --git a/paddle/fluid/framework/fleet/heter_ps/graph_gpu_wrapper.h b/paddle/fluid/framework/fleet/heter_ps/graph_gpu_wrapper.h new file mode 100644 index 0000000000000..6972551b896ed --- /dev/null +++ b/paddle/fluid/framework/fleet/heter_ps/graph_gpu_wrapper.h @@ -0,0 +1,54 @@ +// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include +#include +#include +#include "paddle/fluid/distributed/ps/table/common_graph_table.h" +#include "paddle/fluid/framework/fleet/heter_ps/gpu_graph_node.h" +namespace paddle { +namespace framework { +#ifdef PADDLE_WITH_HETERPS +class GraphGpuWrapper { + public: + char* graph_table; + void initialize(); + void test(); + void set_device(std::vector ids); + void init_service(); + void set_up_types(std::vector& edge_type, + std::vector& node_type); + void upload_batch(int idx, std::vector>& ids); + void add_table_feat_conf(std::string table_name, std::string feat_name, + std::string feat_dtype, int feat_shape); + void load_edge_file(std::string name, std::string filepath, bool reverse); + void load_node_file(std::string name, std::string filepath); + NodeQueryResult query_node_list(int gpu_id, int start, int query_size); + NeighborSampleResult graph_neighbor_sample_v3(NeighborSampleQuery q, + bool cpu_switch); + std::vector graph_neighbor_sample(int gpu_id, + std::vector& key, + int sample_size); + std::unordered_map edge_to_id, feature_to_id; + std::vector id_to_feature, id_to_edge; + std::vector> table_feat_mapping; + std::vector> table_feat_conf_feat_name; + std::vector> table_feat_conf_feat_dtype; + std::vector> table_feat_conf_feat_shape; + ::paddle::distributed::GraphParameter table_proto; + std::vector device_id_mapping; +}; +#endif +} +}; diff --git a/paddle/fluid/framework/fleet/heter_ps/heter_comm_inl.h b/paddle/fluid/framework/fleet/heter_ps/heter_comm_inl.h index 870bad8d19a6f..51432e9de81fb 100644 --- a/paddle/fluid/framework/fleet/heter_ps/heter_comm_inl.h +++ b/paddle/fluid/framework/fleet/heter_ps/heter_comm_inl.h @@ -193,6 +193,8 @@ void HeterComm::walk_to_dest(int start_index, memory_copy(dst_place, node.key_storage, src_place, reinterpret_cast(src_key + h_left[i]), node.key_bytes_len, node.in_stream); + cudaMemsetAsync(node.val_storage, -1, node.val_bytes_len, node.in_stream); + if (need_copy_val) { memory_copy(dst_place, node.val_storage, src_place, reinterpret_cast(src_val + h_left[i]), diff --git a/paddle/fluid/framework/fleet/heter_ps/test_cpu_query.cu b/paddle/fluid/framework/fleet/heter_ps/test_cpu_query.cu index d812542f17ba0..f35a1c41bbe1d 100644 --- a/paddle/fluid/framework/fleet/heter_ps/test_cpu_query.cu +++ b/paddle/fluid/framework/fleet/heter_ps/test_cpu_query.cu @@ -27,6 +27,41 @@ namespace platform = paddle::platform; // paddle::framework::GpuPsCommGraph GraphTable::make_gpu_ps_graph // paddle::framework::GpuPsCommGraph GraphTable::make_gpu_ps_graph( // std::vector ids) + +std::string nodes[] = { + std::string("user\t37\ta 0.34\tb 13 14\tc hello\td abc"), + std::string("user\t96\ta 0.31\tb 15 10\tc 96hello\td abcd"), + std::string("user\t59\ta 0.11\tb 11 14"), + std::string("user\t97\ta 0.11\tb 12 11"), + std::string("item\t45\ta 0.21"), + std::string("item\t145\ta 0.21"), + std::string("item\t112\ta 0.21"), + std::string("item\t48\ta 0.21"), + std::string("item\t247\ta 0.21"), + std::string("item\t111\ta 0.21"), + std::string("item\t46\ta 0.21"), + std::string("item\t146\ta 0.21"), + std::string("item\t122\ta 0.21"), + std::string("item\t49\ta 0.21"), + std::string("item\t248\ta 0.21"), + std::string("item\t113\ta 0.21")}; +char node_file_name[] = "nodes.txt"; +std::vector user_feature_name = {"a", "b", "c", "d"}; +std::vector item_feature_name = {"a"}; +std::vector user_feature_dtype = {"float32", "int32", "string", + "string"}; +std::vector item_feature_dtype = {"float32"}; +std::vector user_feature_shape = {1, 2, 1, 1}; +std::vector item_feature_shape = {1}; +void prepare_file(char file_name[]) { + std::ofstream ofile; + ofile.open(file_name); + + for (auto x : nodes) { + ofile << x << std::endl; + } + ofile.close(); +} TEST(TEST_FLEET, test_cpu_cache) { int gpu_num = 0; int st = 0, u = 0; @@ -34,49 +69,87 @@ TEST(TEST_FLEET, test_cpu_cache) { for (int i = 0; i < 2; i++) device_id_mapping.push_back(i); gpu_num = device_id_mapping.size(); ::paddle::distributed::GraphParameter table_proto; + table_proto.add_edge_types("u2u"); + table_proto.add_node_types("user"); + table_proto.add_node_types("item"); + ::paddle::distributed::GraphFeature *g_f = table_proto.add_graph_feature(); + + for (int i = 0; i < user_feature_name.size(); i++) { + g_f->add_name(user_feature_name[i]); + g_f->add_dtype(user_feature_dtype[i]); + g_f->add_shape(user_feature_shape[i]); + } + ::paddle::distributed::GraphFeature *g_f1 = table_proto.add_graph_feature(); + for (int i = 0; i < item_feature_name.size(); i++) { + g_f1->add_name(item_feature_name[i]); + g_f1->add_dtype(item_feature_dtype[i]); + g_f1->add_shape(item_feature_shape[i]); + } + prepare_file(node_file_name); table_proto.set_shard_num(24); + std::shared_ptr resource = std::make_shared(device_id_mapping); resource->enable_p2p(); int use_nv = 1; GpuPsGraphTable g(resource, use_nv); g.init_cpu_table(table_proto); + g.cpu_graph_table->Load(node_file_name, "nuser"); + g.cpu_graph_table->Load(node_file_name, "nitem"); + std::remove(node_file_name); std::vector vec; + std::vector node_ids; + node_ids.push_back(37); + node_ids.push_back(96); + std::vector> node_feat(2, + std::vector(2)); + std::vector feature_names; + feature_names.push_back(std::string("c")); + feature_names.push_back(std::string("d")); + g.cpu_graph_table->get_node_feat(0, node_ids, feature_names, node_feat); + VLOG(0) << "get_node_feat: " << node_feat[0][0]; + VLOG(0) << "get_node_feat: " << node_feat[0][1]; + VLOG(0) << "get_node_feat: " << node_feat[1][0]; + VLOG(0) << "get_node_feat: " << node_feat[1][1]; int n = 10; std::vector ids0, ids1; for (int i = 0; i < n; i++) { - g.cpu_graph_table->add_comm_edge(i, (i + 1) % n); - g.cpu_graph_table->add_comm_edge(i, (i - 1 + n) % n); + g.cpu_graph_table->add_comm_edge(0, i, (i + 1) % n); + g.cpu_graph_table->add_comm_edge(0, i, (i - 1 + n) % n); if (i % 2 == 0) ids0.push_back(i); } + g.cpu_graph_table->build_sampler(0); ids1.push_back(5); - vec.push_back(g.cpu_graph_table->make_gpu_ps_graph(ids0)); - vec.push_back(g.cpu_graph_table->make_gpu_ps_graph(ids1)); + vec.push_back(g.cpu_graph_table->make_gpu_ps_graph(0, ids0)); + vec.push_back(g.cpu_graph_table->make_gpu_ps_graph(0, ids1)); vec[0].display_on_cpu(); vec[1].display_on_cpu(); g.build_graph_from_cpu(vec); int64_t cpu_key[3] = {0, 1, 2}; + /* + std::vector> buffers(3); + std::vector actual_sizes(3,0); + g.cpu_graph_table->random_sample_neighbors(cpu_key,2,buffers,actual_sizes,false); + for(int i = 0;i < 3;i++){ + VLOG(0)<<"sample from cpu key->"<val, 3 * 2 * sizeof(int64_t), - cudaMemcpyDeviceToHost); - int *actual_sample_size = new int[3]; - cudaMemcpy(actual_sample_size, neighbor_sample_res->actual_sample_size, - 3 * sizeof(int), - cudaMemcpyDeviceToHost); // 3, 1, 3 - - //{0,9} or {9,0} is expected for key 0 + auto neighbor_sample_res = + g.graph_neighbor_sample_v2(0, (int64_t *)key, 2, 3, true); + neighbor_sample_res.display(); + //{1,9} or {9,1} is expected for key 0 //{0,2} or {2,0} is expected for key 1 //{1,3} or {3,1} is expected for key 2 - for (int i = 0; i < 3; i++) { - VLOG(0) << "actual sample size for " << i << " is " - << actual_sample_size[i]; - for (int j = 0; j < actual_sample_size[i]; j++) { - VLOG(0) << "sampled an neighbor for node" << i << " : " << res[i * 2 + j]; - } - } + auto node_query_res = g.query_node_list(0, 0, 4); + node_query_res.display(); + NeighborSampleQuery query; + query.initialize(0, node_query_res.get_val(), 2, node_query_res.get_len()); + query.display(); + auto c = g.graph_neighbor_sample_v3(query, false); + c.display(); } diff --git a/paddle/fluid/framework/fleet/heter_ps/test_sample_rate.cu b/paddle/fluid/framework/fleet/heter_ps/test_sample_rate.cu index 07e561fb3b050..affa60d022ece 100644 --- a/paddle/fluid/framework/fleet/heter_ps/test_sample_rate.cu +++ b/paddle/fluid/framework/fleet/heter_ps/test_sample_rate.cu @@ -264,6 +264,8 @@ void testSampleRate() { res[i].push_back(result); } */ + + // g.graph_neighbor_sample start = 0; auto func = [&rwlock, &g, &start, &ids](int i) { int st = 0; @@ -288,8 +290,37 @@ void testSampleRate() { auto end1 = std::chrono::steady_clock::now(); auto tt = std::chrono::duration_cast(end1 - start1); - std::cerr << "total time cost without cache is " + std::cerr << "total time cost without cache for v1 is " << tt.count() / exe_count / gpu_num1 << " us" << std::endl; + + // g.graph_neighbor_sample_v2 + start = 0; + auto func2 = [&rwlock, &g, &start, &ids](int i) { + int st = 0; + int size = ids.size(); + for (int k = 0; k < exe_count; k++) { + st = 0; + while (st < size) { + int len = std::min(fixed_key_size, (int)ids.size() - st); + auto r = g.graph_neighbor_sample_v2(i, (int64_t *)(key[i] + st), + sample_size, len, false); + st += len; + delete r; + } + } + }; + auto start2 = std::chrono::steady_clock::now(); + std::thread thr2[gpu_num1]; + for (int i = 0; i < gpu_num1; i++) { + thr2[i] = std::thread(func2, i); + } + for (int i = 0; i < gpu_num1; i++) thr2[i].join(); + auto end2 = std::chrono::steady_clock::now(); + auto tt2 = + std::chrono::duration_cast(end2 - start2); + std::cerr << "total time cost without cache for v2 is " + << tt2.count() / exe_count / gpu_num1 << " us" << std::endl; + for (int i = 0; i < gpu_num1; i++) { cudaFree(key[i]); } diff --git a/paddle/fluid/framework/infershape_utils.cc b/paddle/fluid/framework/infershape_utils.cc index bd71ade7e9311..01e594a176bd0 100644 --- a/paddle/fluid/framework/infershape_utils.cc +++ b/paddle/fluid/framework/infershape_utils.cc @@ -28,6 +28,7 @@ limitations under the License. */ #include "paddle/phi/core/compat/op_utils.h" #include "paddle/phi/core/dense_tensor.h" #include "paddle/phi/core/infermeta_utils.h" +#include "paddle/phi/core/kernel_factory.h" #include "paddle/phi/core/tensor_utils.h" namespace paddle { @@ -69,6 +70,11 @@ class InferShapeArgumentMappingContext : public phi::ArgumentMappingContext { } bool IsDenseTensorInput(const std::string& name) const override { + auto var_type = ctx_.GetInputVarType(name); + return var_type == proto::VarType::LOD_TENSOR; + } + + bool IsDenseTensorInputs(const std::string& name) const override { auto var_types = ctx_.GetInputsVarType(name); return std::all_of(var_types.begin(), var_types.end(), [](const proto::VarType::Type& type) { @@ -77,11 +83,8 @@ class InferShapeArgumentMappingContext : public phi::ArgumentMappingContext { } bool IsSelectedRowsInput(const std::string& name) const override { - auto var_types = ctx_.GetInputsVarType(name); - return std::all_of(var_types.begin(), var_types.end(), - [](const proto::VarType::Type& type) { - return type == proto::VarType::SELECTED_ROWS; - }); + auto var_type = ctx_.GetInputVarType(name); + return var_type == proto::VarType::SELECTED_ROWS; } bool IsDenseTensorVectorInput(const std::string& name) const override { @@ -320,7 +323,7 @@ void CompatInferMetaContext::EmplaceBackOutput(CompatMetaTensor output) { } void CompatInferMetaContext::EmplaceBackInputs( - paddle::SmallVector inputs) { + paddle::small_vector inputs) { int index = compat_inputs_.size(); input_range_.emplace_back(std::pair(index, index + inputs.size())); compat_inputs_.insert(compat_inputs_.end(), @@ -329,7 +332,7 @@ void CompatInferMetaContext::EmplaceBackInputs( } void CompatInferMetaContext::EmplaceBackOutputs( - paddle::SmallVector + paddle::small_vector outputs) { int index = compat_outputs_.size(); output_range_.emplace_back( @@ -402,21 +405,20 @@ std::vector CompatInferMetaContext::MutableOutputBetween( CompatInferMetaContext BuildInferMetaContext(InferShapeContext* ctx, const std::string& op_type) { // 1. get kernel args - auto arg_map_fn = phi::OpUtilsMap::Instance().GetArgumentMappingFn(op_type); - PADDLE_ENFORCE_NOT_NULL( - arg_map_fn, platform::errors::NotFound( - "The ArgumentMappingFn of %s op is not found.", op_type)); + auto* arg_map_fn = ctx->GetPhiArgumentMappingFn(); InferShapeArgumentMappingContext arg_map_context(*ctx); - auto signature = arg_map_fn(arg_map_context); + phi::KernelSignature signature = arg_map_fn + ? (*arg_map_fn)(arg_map_context) + : *ctx->GetPhiDefaultKernelSignature(); VLOG(3) << "BuildInferMetaContext: op kernel signature - " << signature; // 2. build infermeta context CompatInferMetaContext infer_meta_context( {ctx->IsRuntime(), ctx->IsRunMKLDNNKernel()}); - auto& input_names = std::get<0>(signature.args); - auto& attr_names = std::get<1>(signature.args); - auto& output_names = std::get<2>(signature.args); + const auto& input_names = signature.input_names; + const auto& attr_names = signature.attr_names; + const auto& output_names = signature.output_names; const auto& args_def = phi::KernelFactory::Instance().GetFirstKernelArgsDef(signature.name); @@ -429,7 +431,7 @@ CompatInferMetaContext BuildInferMetaContext(InferShapeContext* ctx, infer_meta_context.EmplaceBackInput( std::move(CompatMetaTensor(input_var[0], ctx->IsRuntime()))); } else { - paddle::SmallVector + paddle::small_vector inputs; for (const auto& in : input_var) { inputs.emplace_back( @@ -448,7 +450,7 @@ CompatInferMetaContext BuildInferMetaContext(InferShapeContext* ctx, auto attr_reader = ctx->Attrs(); for (size_t i = 0; i < attr_names.size(); ++i) { auto& attr_name = attr_names[i]; - if (attr_defs[i].type_index == std::type_index(typeid(phi::IntArray))) { + if (attr_defs[i].type_index == phi::AttributeType::INT_ARRAY) { // When attr is a vector_tensor or tensor, transform it to IntArray if (ctx->HasInputs(attr_name) || ctx->HasInput(attr_name)) { auto infershape_inputs = std::move(ctx->GetInputVarPtrs(attr_name)); @@ -499,16 +501,13 @@ CompatInferMetaContext BuildInferMetaContext(InferShapeContext* ctx, } } else if (ctx->HasAttr(attr_name)) { auto& attr = attr_reader.GetAttr(attr_name); - if (std::type_index(attr.type()) == - std::type_index(typeid(std::vector))) { + if (AttrTypeID(attr) == proto::AttrType::INTS) { infer_meta_context.EmplaceBackAttr(std::move( phi::IntArray(BOOST_GET_CONST(std::vector, attr)))); - } else if (std::type_index(attr.type()) == - std::type_index(typeid(std::vector))) { + } else if (AttrTypeID(attr) == proto::AttrType::LONGS) { infer_meta_context.EmplaceBackAttr(std::move( phi::IntArray(BOOST_GET_CONST(std::vector, attr)))); - } else if (std::type_index(attr.type()) == - std::type_index(typeid(int))) { + } else if (AttrTypeID(attr) == proto::AttrType::INT) { infer_meta_context.EmplaceBackAttr( phi::IntArray({BOOST_GET_CONST(int, attr)})); } else { @@ -518,20 +517,17 @@ CompatInferMetaContext BuildInferMetaContext(InferShapeContext* ctx, attr_name)); } } - } else if (attr_defs[i].type_index == - std::type_index(typeid(phi::Scalar))) { + } else if (attr_defs[i].type_index == phi::AttributeType::SCALAR) { if (ctx->HasAttr(attr_name)) { // TODO(chentianyu03): support other attrs later auto& attr = attr_reader.GetAttr(attr_name); - if (std::type_index(attr.type()) == std::type_index(typeid(float))) { + if (AttrTypeID(attr) == proto::AttrType::FLOAT) { infer_meta_context.EmplaceBackAttr( phi::Scalar(BOOST_GET_CONST(float, attr))); - } else if (std::type_index(attr.type()) == - std::type_index(typeid(std::string))) { + } else if (AttrTypeID(attr) == proto::AttrType::STRING) { infer_meta_context.EmplaceBackAttr( phi::Scalar(BOOST_GET_CONST(std::string, attr))); - } else if (std::type_index(attr.type()) == - std::type_index(typeid(int))) { + } else if (AttrTypeID(attr) == proto::AttrType::INT) { infer_meta_context.EmplaceBackAttr( phi::Scalar(BOOST_GET_CONST(int, attr))); } else { @@ -559,11 +555,9 @@ CompatInferMetaContext BuildInferMetaContext(InferShapeContext* ctx, attr_name, infershape_input.size())); } } - } else if (attr_defs[i].type_index == - std::type_index(typeid(std::vector))) { + } else if (attr_defs[i].type_index == phi::AttributeType::SCALARS) { auto& attr = attr_reader.GetAttr(attr_name); - if (std::type_index(attr.type()) == - std::type_index(typeid(std::vector))) { + if (AttrTypeID(attr) == proto::AttrType::INTS) { const auto& vec = BOOST_GET_CONST(std::vector, attr); std::vector scalar_list; scalar_list.reserve(vec.size()); @@ -571,8 +565,7 @@ CompatInferMetaContext BuildInferMetaContext(InferShapeContext* ctx, scalar_list.emplace_back(val); } infer_meta_context.EmplaceBackAttr(std::move(scalar_list)); - } else if (std::type_index(attr.type()) == - std::type_index(typeid(std::vector))) { + } else if (AttrTypeID(attr) == proto::AttrType::LONGS) { const auto& vec = BOOST_GET_CONST(std::vector, attr); std::vector scalar_list; scalar_list.reserve(vec.size()); @@ -580,8 +573,7 @@ CompatInferMetaContext BuildInferMetaContext(InferShapeContext* ctx, scalar_list.emplace_back(val); } infer_meta_context.EmplaceBackAttr(std::move(scalar_list)); - } else if (std::type_index(attr.type()) == - std::type_index(typeid(std::vector))) { + } else if (AttrTypeID(attr) == proto::AttrType::FLOATS) { const auto& vec = BOOST_GET_CONST(std::vector, attr); std::vector scalar_list; scalar_list.reserve(vec.size()); @@ -589,8 +581,7 @@ CompatInferMetaContext BuildInferMetaContext(InferShapeContext* ctx, scalar_list.emplace_back(val); } infer_meta_context.EmplaceBackAttr(std::move(scalar_list)); - } else if (std::type_index(attr.type()) == - std::type_index(typeid(std::vector))) { + } else if (AttrTypeID(attr) == proto::AttrType::FLOAT64S) { const auto& vec = BOOST_GET_CONST(std::vector, attr); std::vector scalar_list; scalar_list.reserve(vec.size()); @@ -607,29 +598,24 @@ CompatInferMetaContext BuildInferMetaContext(InferShapeContext* ctx, } else if (ctx->HasAttr(attr_name)) { // Emplace Back Attr according to the type of attr. auto& attr = attr_reader.GetAttr(attr_name); - if (attr_defs[i].type_index == std::type_index(typeid(bool))) { + if (attr_defs[i].type_index == phi::AttributeType::BOOL) { infer_meta_context.EmplaceBackAttr(BOOST_GET_CONST(bool, attr)); - } else if (attr_defs[i].type_index == std::type_index(typeid(int))) { + } else if (attr_defs[i].type_index == phi::AttributeType::INT32) { infer_meta_context.EmplaceBackAttr(BOOST_GET_CONST(int, attr)); - } else if (attr_defs[i].type_index == std::type_index(typeid(int64_t))) { + } else if (attr_defs[i].type_index == phi::AttributeType::INT64) { infer_meta_context.EmplaceBackAttr(BOOST_GET_CONST(int64_t, attr)); - } else if (attr_defs[i].type_index == std::type_index(typeid(float))) { + } else if (attr_defs[i].type_index == phi::AttributeType::FLOAT32) { infer_meta_context.EmplaceBackAttr(BOOST_GET_CONST(float, attr)); - } else if (attr_defs[i].type_index == - std::type_index(typeid(std::string))) { + } else if (attr_defs[i].type_index == phi::AttributeType::STRING) { infer_meta_context.EmplaceBackAttr(BOOST_GET_CONST(std::string, attr)); - } else if (attr_defs[i].type_index == - std::type_index(typeid(std::vector))) { + } else if (attr_defs[i].type_index == phi::AttributeType::BOOLS) { infer_meta_context.EmplaceBackAttr( BOOST_GET_CONST(std::vector, attr)); - } else if (attr_defs[i].type_index == - std::type_index(typeid(std::vector))) { + } else if (attr_defs[i].type_index == phi::AttributeType::INT32S) { infer_meta_context.EmplaceBackAttr( BOOST_GET_CONST(std::vector, attr)); - } else if (attr_defs[i].type_index == - std::type_index(typeid(std::vector))) { - if (std::type_index(attr.type()) == - std::type_index(typeid(std::vector))) { + } else if (attr_defs[i].type_index == phi::AttributeType::INT64S) { + if (AttrTypeID(attr) == proto::AttrType::INTS) { // Emplace Back Attr according to the type of Phi_Kernel args. const auto& vector_int_attr = BOOST_GET_CONST(std::vector, attr); const std::vector vector_int64_attr(vector_int_attr.begin(), @@ -639,20 +625,16 @@ CompatInferMetaContext BuildInferMetaContext(InferShapeContext* ctx, infer_meta_context.EmplaceBackAttr( BOOST_GET_CONST(std::vector, attr)); } - } else if (attr_defs[i].type_index == - std::type_index(typeid(std::vector))) { + } else if (attr_defs[i].type_index == phi::AttributeType::FLOAT32S) { infer_meta_context.EmplaceBackAttr( BOOST_GET_CONST(std::vector, attr)); - } else if (attr_defs[i].type_index == - std::type_index(typeid(std::vector))) { + } else if (attr_defs[i].type_index == phi::AttributeType::FLOAT64S) { infer_meta_context.EmplaceBackAttr( BOOST_GET_CONST(std::vector, attr)); - } else if (attr_defs[i].type_index == - std::type_index(typeid(std::vector))) { + } else if (attr_defs[i].type_index == phi::AttributeType::STRINGS) { infer_meta_context.EmplaceBackAttr( BOOST_GET_CONST(std::vector, attr)); - } else if (attr_defs[i].type_index == - std::type_index(typeid(phi::DataType))) { + } else if (attr_defs[i].type_index == phi::AttributeType::DATA_TYPE) { auto data_type = paddle::framework::TransToPhiDataType( static_cast( BOOST_GET_CONST(int, attr))); @@ -664,7 +646,7 @@ CompatInferMetaContext BuildInferMetaContext(InferShapeContext* ctx, } } else if (ctx->HasInput(attr_name)) { // convert from data - if (attr_defs[i].type_index == std::type_index(typeid(int32_t))) { + if (attr_defs[i].type_index == phi::AttributeType::INT32) { if (ctx->IsRuntime()) { auto infershape_inputs = std::move(ctx->GetInputVarPtrs(attr_name)); auto var_temp = BOOST_GET_CONST(Variable*, infershape_inputs[i]); @@ -690,7 +672,7 @@ CompatInferMetaContext BuildInferMetaContext(InferShapeContext* ctx, infer_meta_context.EmplaceBackOutput( std::move(CompatMetaTensor(output_var[0], ctx->IsRuntime()))); } else { - paddle::SmallVector + paddle::small_vector outputs; for (const auto& out : output_var) { if (ctx->IsRuntime()) { diff --git a/paddle/fluid/framework/infershape_utils.h b/paddle/fluid/framework/infershape_utils.h index e54f2e81e7e9f..855e873b30951 100644 --- a/paddle/fluid/framework/infershape_utils.h +++ b/paddle/fluid/framework/infershape_utils.h @@ -100,9 +100,10 @@ class CompatInferMetaContext : public phi::InferMetaContext { void EmplaceBackOutput(CompatMetaTensor output); void EmplaceBackInputs( - paddle::SmallVector inputs); + paddle::small_vector + inputs); void EmplaceBackOutputs( - paddle::SmallVector + paddle::small_vector outputs); const phi::MetaTensor& InputAt(size_t idx) const override; @@ -121,9 +122,9 @@ class CompatInferMetaContext : public phi::InferMetaContext { virtual ~CompatInferMetaContext() = default; private: - paddle::SmallVector + paddle::small_vector compat_inputs_; - paddle::SmallVector + paddle::small_vector compat_outputs_; }; diff --git a/paddle/fluid/framework/ir/CMakeLists.txt b/paddle/fluid/framework/ir/CMakeLists.txt index 207ee713bf409..a2f3b8dc7911a 100755 --- a/paddle/fluid/framework/ir/CMakeLists.txt +++ b/paddle/fluid/framework/ir/CMakeLists.txt @@ -226,6 +226,7 @@ endif() cc_test(test_cpu_quantize_squash_pass SRCS mkldnn/cpu_quantize_squash_pass_tester.cc DEPS cpu_quantize_squash_pass naive_executor) cc_test(test_reshape_transpose_matmul_mkldnn_fuse_pass SRCS mkldnn/reshape_transpose_matmul_mkldnn_fuse_pass_tester.cc DEPS reshape_transpose_matmul_mkldnn_fuse_pass reshape_transpose_matmul_v2_mkldnn_fuse_pass) cc_test(test_matmul_transpose_reshape_fuse_pass SRCS mkldnn/matmul_transpose_reshape_fuse_pass_tester.cc DEPS matmul_transpose_reshape_fuse_pass matmul_v2_transpose_reshape_fuse_pass) + cc_test(test_shuffle_channel_mkldnn_detect_pass SRCS mkldnn/shuffle_channel_mkldnn_detect_pass_tester.cc DEPS shuffle_channel_mkldnn_detect_pass) cc_test(test_cpu_bfloat16_placement_pass SRCS mkldnn/cpu_bfloat16_placement_pass_tester.cc DEPS cpu_bfloat16_placement_pass) cc_test(test_cpu_bfloat16_pass SRCS mkldnn/cpu_bfloat16_pass_tester.cc DEPS cpu_bfloat16_pass) cc_test(test_multi_gru_fuse_pass SRCS mkldnn/multi_gru_fuse_pass_tester.cc DEPS multi_gru_fuse_pass) diff --git a/paddle/fluid/framework/ir/fusion_group/code_generator_tester.cc b/paddle/fluid/framework/ir/fusion_group/code_generator_tester.cc index f671e0ae7690a..7b6bbf0251001 100644 --- a/paddle/fluid/framework/ir/fusion_group/code_generator_tester.cc +++ b/paddle/fluid/framework/ir/fusion_group/code_generator_tester.cc @@ -420,11 +420,7 @@ std::unique_ptr BuildGraph(bool backward, n->Var()->SetDataType(proto_dtype); } } -#ifdef __clang__ return graph; -#else - return std::move(graph); -#endif } std::unordered_set DistilGradNodes( diff --git a/paddle/fluid/framework/ir/fusion_group/fusion_group_pass_tester.cc b/paddle/fluid/framework/ir/fusion_group/fusion_group_pass_tester.cc index d14c7e433bd08..db22c03a7d9c0 100644 --- a/paddle/fluid/framework/ir/fusion_group/fusion_group_pass_tester.cc +++ b/paddle/fluid/framework/ir/fusion_group/fusion_group_pass_tester.cc @@ -63,11 +63,7 @@ std::unique_ptr BuildElementwiseListGraph(bool backward = false) { n->Var()->SetDataType(proto::VarType::FP32); } } -#ifdef __clang__ return graph; -#else - return std::move(graph); -#endif } std::unique_ptr BuildElementwiseTreeGraph(bool backward = false) { @@ -125,11 +121,7 @@ std::unique_ptr BuildElementwiseTreeGraph(bool backward = false) { n->Var()->SetDataType(proto::VarType::FP32); } } -#ifdef __clang__ return graph; -#else - return std::move(graph); -#endif } int TestMain(std::unique_ptr graph, std::string prefix) { diff --git a/paddle/fluid/framework/ir/graph_pattern_detector.cc b/paddle/fluid/framework/ir/graph_pattern_detector.cc index 8eb1b64a2763a..fbd8fda131b6d 100644 --- a/paddle/fluid/framework/ir/graph_pattern_detector.cc +++ b/paddle/fluid/framework/ir/graph_pattern_detector.cc @@ -2665,41 +2665,8 @@ PDNode *patterns::UnsupportedBfloat16::operator()() { return op; } -PDNode *patterns::LastBfloat16Ops::operator()() { - auto *op = pattern->NewNode(op_repr())->assert_is_op(); - op->assert_more([&](Node *node) { - return node->Op()->GetAttrIfExists("mkldnn_data_type") == - "bfloat16"; - }); - auto *op_out = pattern->NewNode(op_out_repr())->AsOutput(); - op->LinksTo({op_out}); - return op_out; -} - -PDNode *patterns::FirstBfloat16Ops::operator()() { - auto *op_in = pattern->NewNode(op_in_repr())->AsInput(); - - auto *op = pattern->NewNode(op_repr())->assert_is_op(); - op->assert_more([&](Node *node) { - return node->Op()->GetAttrIfExists("mkldnn_data_type") == - "bfloat16"; - }); - - op->LinksFrom({op_in}); - return op; -} - -PDNode *patterns::DuplicatedInputs::operator()() { - auto op = pattern->NewNode(op_repr())->assert_is_ops({"concat", "sum"}); - op->assert_more([&](Node *node) { - return node->Op()->GetAttrIfExists("mkldnn_data_type") == - "bfloat16"; - }); - return op; -} - -PDNode *patterns::DuplicatedOutputs::operator()() { - auto op = pattern->NewNode(op_repr())->assert_is_ops({"split"}); +PDNode *patterns::Bloat16Ops::operator()() { + auto op = pattern->NewNode(op_repr())->assert_is_op(); op->assert_more([&](Node *node) { return node->Op()->GetAttrIfExists("mkldnn_data_type") == "bfloat16"; diff --git a/paddle/fluid/framework/ir/graph_pattern_detector.h b/paddle/fluid/framework/ir/graph_pattern_detector.h index 434ede6cf7a3b..d7e265fe28bf9 100644 --- a/paddle/fluid/framework/ir/graph_pattern_detector.h +++ b/paddle/fluid/framework/ir/graph_pattern_detector.h @@ -1565,36 +1565,9 @@ struct UnsupportedBfloat16 : public PatternBase { PATTERN_DECL_NODE(op); }; -struct LastBfloat16Ops : public PatternBase { - LastBfloat16Ops(PDPattern* pattern, const std::string& name_scope) - : PatternBase(pattern, name_scope, "last_bfloat16_ops") {} - PDNode* operator()(); - - PATTERN_DECL_NODE(op); - PATTERN_DECL_NODE(op_out); -}; - -struct FirstBfloat16Ops : public PatternBase { - FirstBfloat16Ops(PDPattern* pattern, const std::string& name_scope) - : PatternBase(pattern, name_scope, "first_bfloat16_ops") {} - PDNode* operator()(); - - PATTERN_DECL_NODE(op_in); - PATTERN_DECL_NODE(op); -}; - -struct DuplicatedInputs : public PatternBase { - DuplicatedInputs(PDPattern* pattern, const std::string& name_scope) - : PatternBase(pattern, name_scope, "many_inputs_op") {} - - PDNode* operator()(); - - PATTERN_DECL_NODE(op); -}; - -struct DuplicatedOutputs : public PatternBase { - DuplicatedOutputs(PDPattern* pattern, const std::string& name_scope) - : PatternBase(pattern, name_scope, "many_outputs_op") {} +struct Bloat16Ops : public PatternBase { + Bloat16Ops(PDPattern* pattern, const std::string& name_scope) + : PatternBase(pattern, name_scope, "many_bfloat16_ops") {} PDNode* operator()(); diff --git a/paddle/fluid/framework/ir/memory_optimize_pass/inplace_addto_op_pass.cc b/paddle/fluid/framework/ir/memory_optimize_pass/inplace_addto_op_pass.cc index 0ed2ec51b89cb..680dad5cc6b20 100644 --- a/paddle/fluid/framework/ir/memory_optimize_pass/inplace_addto_op_pass.cc +++ b/paddle/fluid/framework/ir/memory_optimize_pass/inplace_addto_op_pass.cc @@ -178,9 +178,11 @@ void InplaceAddToOpPass::Run(Graph *graph) const { auto *out_generated_op = dynamic_cast( out_var_ptr->GeneratedOp()); - // NOTE(zhiqiu): currently, only conv2d_grad supports addto strategy + // FIXME(zengjinle): the "custom_fused_dense_grad" is only used for + // MLPerf temporarily. Replace it with the formal op type in the future. if (right_generated_op->Name() != "conv2d_grad" && - right_generated_op->Name() != "resnet_unit_grad") { + right_generated_op->Name() != "resnet_unit_grad" && + right_generated_op->Name() != "custom_fused_dense_grad") { continue; } diff --git a/paddle/fluid/framework/ir/mkldnn/cpu_bfloat16_pass.cc b/paddle/fluid/framework/ir/mkldnn/cpu_bfloat16_pass.cc index f1bd34a5ad4f6..62b2be712beef 100644 --- a/paddle/fluid/framework/ir/mkldnn/cpu_bfloat16_pass.cc +++ b/paddle/fluid/framework/ir/mkldnn/cpu_bfloat16_pass.cc @@ -22,290 +22,226 @@ namespace paddle { namespace framework { namespace ir { -using string::PrettyLogDetail; +namespace { +class Quanter { + public: + void AddQuantOps() { + if (IsNotPermittedOpType()) return; -void UnlinkNodes(ir::Node* a, ir::Node* b) { - a->outputs.erase(std::remove(a->outputs.begin(), a->outputs.end(), b), - a->outputs.end()); - b->inputs.erase(std::remove(b->inputs.begin(), b->inputs.end(), a), - b->inputs.end()); -} + std::vector linked_xputs; -// Checking whether a reorder from FP32 to BF16 should be added before the input -// to the operator -bool IsPermittedInputName(const std::string& input_name) { - // Only the inputs listed in \"permitted_names\" requires quanitization before - // the bfloat16 operator. Other inputs, such as Filter and Bias are reordered - // in the kernel. - const std::vector permitted_names = {"X", "Y", "Input", - "ResidualData"}; - return (std::find(permitted_names.begin(), permitted_names.end(), - input_name) != permitted_names.end()); -} + for (const auto& logical_xput : op_xputs) { + std::vector quant_xput_names; + quant_xput_names.reserve(xputs_map.size()); -// Checking whether a reorder from BF16 to FP32 should be added after the output -// to the operator -bool IsPermittedOutputName(const std::string& output_name) { - // XShape is output in transpose2 and reshape2 operators used to store the - // shape and lod of X. So this output do not need dequantize before. - return (output_name != "XShape"); -} + const auto& logical_xput_name = logical_xput.first; + if (IsNotPermittedName(logical_xput_name)) continue; -void AddQuantize(Graph* g, ir::Node* op, ir::Node* op_in, - int& quantize_counter) { - std::vector input_names; - - // Find the name of the input linking op to op_in - for (auto name : op->Op()->InputNames()) - for (auto input_name : op->Op()->Input(name)) - if (input_name == op_in->Name() && IsPermittedInputName(name)) - input_names.push_back(name); - - if (input_names.empty()) return; - - VarDesc quantize_out_desc(patterns::PDNodeName("quantize", "out")); - auto* quantize_out_node = g->CreateVarNode(&quantize_out_desc); - - OpDesc q_desc; - q_desc.SetType("quantize"); - q_desc.SetInput("Input", std::vector({op_in->Name()})); - q_desc.SetOutput("Output", - std::vector({quantize_out_node->Name()})); - q_desc.SetAttr("Scale", 1.f); - q_desc.SetAttr("Shift", 0.0f); - q_desc.SetAttr("bfloat16", true); - q_desc.SetAttr("output_format", op->Op()->HasAttr("data_layout") - ? op->Op()->GetAttr("data_layout") - : std::string("NCHW")); - auto quantize_op = g->CreateOpNode(&q_desc); // OpDesc will be copied. - - for (auto name = input_names.begin(); name < input_names.end(); name++) - op->Op()->SetInput(*name, - std::vector({quantize_out_node->Name()})); - - UnlinkNodes(op_in, op); - IR_NODE_LINK_TO(op_in, quantize_op); - IR_NODE_LINK_TO(quantize_op, quantize_out_node); - IR_NODE_LINK_TO(quantize_out_node, op); - quantize_counter++; -} + const auto& physical_xputs_names = logical_xput.second; + for (const auto& physical_xput_name : physical_xputs_names) { + if (IsAlreadyLinked(linked_xputs, physical_xput_name)) continue; -void AddQuantizes(Graph* g, ir::Node* op, int& quantize_counter) { - auto inputs = op->inputs; - PADDLE_ENFORCE_GE(inputs.size(), 1, - platform::errors::InvalidArgument( - "OP(%s)'s inputs(%d) must be equal or greater than 1.", - op->Name(), inputs.size())); - PADDLE_ENFORCE_EQ(op->outputs.size(), 1, - platform::errors::InvalidArgument( - "OP(%s)'s outputs(%d) must be equal to 1.", op->Name(), - op->outputs.size())); - - OpDesc q_desc; - q_desc.SetType("quantize"); - - std::vector quantize_out_nodes(inputs.size()); - std::vector quantize_out_node_names(inputs.size()); - - for (size_t i = 0; i < inputs.size(); i++) { - VarDesc quantize_out_desc(patterns::PDNodeName("quantize", "out")); - quantize_out_nodes[i] = g->CreateVarNode(&quantize_out_desc); - quantize_out_node_names[i] = quantize_out_nodes[i]->Name(); - - q_desc.SetInput("Input", std::vector({inputs[i]->Name()})); - q_desc.SetOutput("Output", - std::vector({quantize_out_node_names[i]})); - q_desc.SetAttr("Scale", 1.f); - q_desc.SetAttr("Shift", 0.0f); - q_desc.SetAttr("bfloat16", true); - q_desc.SetAttr("output_format", op->Op()->HasAttr("data_layout") - ? op->Op()->GetAttr("data_layout") - : std::string("NCHW")); - auto quantize_op = g->CreateOpNode(&q_desc); // OpDesc will be copied. - - UnlinkNodes(inputs[i], op); - IR_NODE_LINK_TO(inputs[i], quantize_op); - IR_NODE_LINK_TO(quantize_op, quantize_out_nodes[i]); - IR_NODE_LINK_TO(quantize_out_nodes[i], op); - quantize_counter++; + VarDesc quant_x_desc( + patterns::PDNodeName(get_op_type(), get_op_edge())); + auto quant_x_node = graph.CreateVarNode(&quant_x_desc); + const auto xput_name = quant_x_node->Name(); + quant_xput_names.emplace_back(xput_name); + + auto quant_op = create_quant_op(physical_xput_name, xput_name); + + auto physical_xput_node = xputs_map[physical_xput_name]; + link_nodes(physical_xput_node, quant_op, quant_x_node); + counter++; + linked_xputs.push_back(physical_xput_name); + } + + set_edge(logical_xput_name, quant_xput_names); + } } - op->Op()->SetInput("X", quantize_out_node_names); -} + int get_counter() const { return counter; } -// Operators like Concat and Sum have a single input name X, which actually -// consists of multiple inputs. Such operators require a different way to find -// pattern and add quantize ops. -void AddReoderBeforeDuplicatedInputs(ir::Graph* graph, int& quantize_counter) { - GraphPatternDetector gpd; - patterns::DuplicatedInputs duplicated_inputs{gpd.mutable_pattern(), - "duplicated_inputs"}; - duplicated_inputs(); - auto handler = [&](const GraphPatternDetector::subgraph_t& subgraph, - Graph* g) { - GET_IR_NODE_FROM_SUBGRAPH(op, op, duplicated_inputs); - AddQuantizes(g, op, quantize_counter); + virtual ~Quanter() = default; + + protected: + Graph& graph; + ir::Node* const op; + + std::map xputs_map; + const VariableNameMap& op_xputs; + + int counter = 0; + + Quanter(Graph& graph, ir::Node* const op, const VariableNameMap& op_xputs) + : graph(graph), op(op), op_xputs(op_xputs){}; + + virtual bool IsNotPermittedOpType() const = 0; + virtual bool IsNotPermittedName(const std::string& input_name) const = 0; + virtual std::string get_op_type() const = 0; + virtual std::string get_op_edge() const = 0; + virtual void link_nodes(ir::Node* const physical_xput_node, + ir::Node* const quant_op, + ir::Node* const quant_x_node) = 0; + virtual void set_edge(const std::string& logical_xput_name, + const std::vector& quant_xput_names) = 0; + + bool IsAlreadyLinked(const std::vector& node_names, + const std::string& node_name) const { + return std::find(node_names.begin(), node_names.end(), node_name) != + node_names.end(); + } + + virtual ir::Node* create_quant_op(const std::string& input_name, + const std::string& output_name) const { + OpDesc op_desc; + op_desc.SetType(get_op_type()); + + op_desc.SetInput("Input", std::vector({input_name})); + op_desc.SetOutput("Output", std::vector({output_name})); + op_desc.SetAttr("Scale", 1.f); + op_desc.SetAttr("Shift", 0.0f); + op_desc.SetAttr("bfloat16", true); + op_desc.SetAttr("output_format", op->Op()->HasAttr("data_layout") + ? op->Op()->GetAttr("data_layout") + : std::string("NCHW")); + return graph.CreateOpNode(&op_desc); // OpDesc will be copied. + } + + void UnlinkNodes(ir::Node* a, ir::Node* b) const { + a->outputs.erase(std::remove(a->outputs.begin(), a->outputs.end(), b), + a->outputs.end()); + b->inputs.erase(std::remove(b->inputs.begin(), b->inputs.end(), a), + b->inputs.end()); + } +}; + +class Quantizer final : public Quanter { + public: + Quantizer(Graph* const graph, ir::Node* const op) + : Quanter(*graph, op, op->Op()->Inputs()) { + auto inputs = op->inputs; + PADDLE_ENFORCE_GE( + inputs.size(), 1, + platform::errors::InvalidArgument( + "OP(%s)'s inputs(%d) must be equal or greater than 1.", op->Name(), + inputs.size())); + + for (auto input : inputs) xputs_map[input->Name()] = input; }; - gpd(graph, handler); -} -// Adding quantize ops before all operators except Concat and Sum, which have -// already been handled in AddReoderBeforeDuplicatedInputs -void AddReoderBeforeSingleInputs(ir::Graph* graph, int& quantize_counter) { - GraphPatternDetector gpd; - patterns::FirstBfloat16Ops bfloat16_ops{gpd.mutable_pattern(), - "first_bfloat16_ops"}; - bfloat16_ops(); - auto handler = [&](const GraphPatternDetector::subgraph_t& subgraph, - Graph* g) { - GET_IR_NODE_FROM_SUBGRAPH(op_in, op_in, bfloat16_ops); - GET_IR_NODE_FROM_SUBGRAPH(op, op, bfloat16_ops); - if (op->Op()->Type() != "sum" && op->Op()->Type() != "concat") { - AddQuantize(g, op, op_in, quantize_counter); - } + protected: + bool IsNotPermittedOpType() const override { return false; } + + // Checking whether a reorder from FP32 to BF16 + // should be added before the input to the operator + bool IsNotPermittedName(const std::string& input_name) const override { + // Only the inputs listed in \"permitted_names\" + // requires quanitization before the bfloat16 operator. + // Other inputs, such as Filter and Bias are reordered in the kernel. + const std::vector permitted_names = {"X", "Y", "Input", + "ResidualData"}; + + return std::none_of( + permitted_names.begin(), permitted_names.end(), + [&input_name](const std::string& name) { return name == input_name; }); + } + + std::string get_op_type() const override { return "quantize"; }; + std::string get_op_edge() const override { return "out"; }; + + void link_nodes(ir::Node* const physical_xput_node, ir::Node* const quant_op, + ir::Node* const quant_x_node) override { + UnlinkNodes(physical_xput_node, op); + IR_NODE_LINK_TO(physical_xput_node, quant_op); + IR_NODE_LINK_TO(quant_op, quant_x_node); + IR_NODE_LINK_TO(quant_x_node, op); + } + + void set_edge(const std::string& logical_xput_name, + const std::vector& quant_xput_names) override { + op->Op()->SetInput(logical_xput_name, quant_xput_names); + } +}; + +class DeQuantizer final : public Quanter { + public: + DeQuantizer(Graph* const graph, ir::Node* const op) + : Quanter(*graph, op, op->Op()->Outputs()) { + auto outputs = op->outputs; + PADDLE_ENFORCE_GE( + outputs.size(), 1, + platform::errors::InvalidArgument( + "OP(%s)'s outputs(%d) must be equal or greater than 1.", op->Name(), + outputs.size())); + + for (auto output : outputs) xputs_map[output->Name()] = output; }; - gpd(graph, handler); -} -void CPUBFloat16Pass::SetInputDataType(ir::Graph* graph) const { - int quantize_counter = 0; - AddReoderBeforeDuplicatedInputs(graph, quantize_counter); - AddReoderBeforeSingleInputs(graph, quantize_counter); - PrettyLogDetail("--- added %d quantize ops before bfloat16 op", - quantize_counter); -} + protected: + bool IsNotPermittedOpType() const override { + // Prior_box operator output is always FP32 so no dequantization is needed. + return op->Op()->Type() == "prior_box"; + } -void AddDequantize(Graph* g, ir::Node* op, ir::Node* op_out, - int& dequantize_counter) { - if (op->Op()->Type() == "prior_box") return; - - // Find the name of the output linking op to op_out - std::vector output_names; - for (auto name : op->Op()->OutputNames()) - for (auto output_name : op->Op()->Output(name)) - if (output_name == op_out->Name() && IsPermittedOutputName(name)) - output_names.push_back(name); - - if (output_names.empty()) return; - - VarDesc dequantize_in_desc(patterns::PDNodeName("dequantize", "in")); - auto* dequantize_in_node = g->CreateVarNode(&dequantize_in_desc); - - OpDesc deq_desc; - deq_desc.SetType("dequantize"); - deq_desc.SetInput("Input", - std::vector({dequantize_in_node->Name()})); - deq_desc.SetOutput("Output", std::vector({op_out->Name()})); - deq_desc.SetAttr("Scale", 1.0f); - deq_desc.SetAttr("Shift", 0.0f); - auto dequantize_op = g->CreateOpNode(&deq_desc); // OpDesc will be copied. - - for (auto name = output_names.begin(); name < output_names.end(); name++) - op->Op()->SetOutput(*name, - std::vector({dequantize_in_node->Name()})); - - UnlinkNodes(op, op_out); - IR_NODE_LINK_TO(op, dequantize_in_node); - IR_NODE_LINK_TO(dequantize_in_node, dequantize_op); - IR_NODE_LINK_TO(dequantize_op, op_out); - - dequantize_counter++; -} + // Checking whether a reorder from BF16 to FP32 + // should be added after the output to the operator + bool IsNotPermittedName(const std::string& output_name) const override { + // XShape is output in transpose2 and reshape2 operators used to store the + // shape and lod of X. So this output do not need dequantize before. + return (output_name == "XShape"); + } + + std::string get_op_type() const override { return "dequantize"; }; + std::string get_op_edge() const override { return "in"; }; -void AddDequantizes(Graph* g, ir::Node* op, int& dequantize_counter) { - auto outputs = op->outputs; - PADDLE_ENFORCE_GE(outputs.size(), 1, - platform::errors::InvalidArgument( - "OP(%s)'s outputs(%d) must be equal or greater than 1.", - op->Name(), outputs.size())); - PADDLE_ENFORCE_EQ(op->inputs.size(), 1, - platform::errors::InvalidArgument( - "OP(%s)'s inputs(%d) must be equal to 1.", op->Name(), - op->inputs.size())); - - OpDesc deq_desc; - deq_desc.SetType("dequantize"); - - std::vector dequantize_in_nodes(outputs.size()); - std::vector dequantize_in_node_names(outputs.size()); - - for (size_t i = 0; i < outputs.size(); i++) { - VarDesc dequantize_in_desc(patterns::PDNodeName("dequantize", "in")); - dequantize_in_nodes[i] = g->CreateVarNode(&dequantize_in_desc); - dequantize_in_node_names[i] = dequantize_in_nodes[i]->Name(); - - deq_desc.SetInput("Input", - std::vector({dequantize_in_node_names[i]})); - deq_desc.SetOutput("Output", - std::vector({outputs[i]->Name()})); - - deq_desc.SetAttr("Scale", 1.f); - deq_desc.SetAttr("Shift", 0.0f); - deq_desc.SetAttr("bfloat16", true); - deq_desc.SetAttr("output_format", op->Op()->HasAttr("data_layout") - ? op->Op()->GetAttr("data_layout") - : std::string("NCHW")); - auto dequantize_op = g->CreateOpNode(&deq_desc); // OpDesc will be copied. - - UnlinkNodes(op, outputs[i]); - IR_NODE_LINK_TO(op, dequantize_in_nodes[i]); - IR_NODE_LINK_TO(dequantize_in_nodes[i], dequantize_op); - IR_NODE_LINK_TO(dequantize_op, outputs[i]); - - dequantize_counter++; + void link_nodes(ir::Node* const physical_xput_node, ir::Node* const quant_op, + ir::Node* const quant_x_node) override { + UnlinkNodes(op, physical_xput_node); + IR_NODE_LINK_TO(quant_op, physical_xput_node); + IR_NODE_LINK_TO(quant_x_node, quant_op); + IR_NODE_LINK_TO(op, quant_x_node); } - op->Op()->SetOutput("Out", dequantize_in_node_names); -} + void set_edge(const std::string& logical_xput_name, + const std::vector& quant_xput_names) override { + op->Op()->SetOutput(logical_xput_name, quant_xput_names); + } -// Operators like split have a single output name Out, which actually -// consists of multiple outputs. Such operators require a different way to find -// pattern and add dequantize ops. -void AddReoderAfterDuplicatedOutputs(ir::Graph* graph, - int& dequantize_counter) { - GraphPatternDetector gpd; - patterns::DuplicatedOutputs duplicated_outputs{gpd.mutable_pattern(), - "duplicated_outputs"}; - duplicated_outputs(); - auto handler = [&](const GraphPatternDetector::subgraph_t& subgraph, - Graph* g) { - GET_IR_NODE_FROM_SUBGRAPH(op, op, duplicated_outputs); - AddDequantizes(g, op, dequantize_counter); - }; - gpd(graph, handler); + ir::Node* create_quant_op(const std::string& input_name, + const std::string& output_name) const override { + return Quanter::create_quant_op(output_name, input_name); + } +}; } +using string::PrettyLogDetail; + +void CPUBFloat16Pass::ApplyImpl(ir::Graph* graph) const { + int quantize_counter = 0; + int dequantize_counter = 0; -// Adding dequantize ops after all operators except split, which has -// already been handled in AddReoderAfterDuplicatedOutputs -void AddReoderAfterSingleOutputs(ir::Graph* graph, int& dequantize_counter) { GraphPatternDetector gpd; - patterns::LastBfloat16Ops bfloat16_ops{gpd.mutable_pattern(), - "last_bfloat16_ops"}; - bfloat16_ops(); + patterns::Bloat16Ops Bloat16Ops{gpd.mutable_pattern(), "Bloat16Ops"}; + Bloat16Ops(); auto handler = [&](const GraphPatternDetector::subgraph_t& subgraph, - Graph* g) { - GET_IR_NODE_FROM_SUBGRAPH(op_out, op_out, bfloat16_ops); - GET_IR_NODE_FROM_SUBGRAPH(op, op, bfloat16_ops); - if (op->Op()->Type() != "split") { - AddDequantize(g, op, op_out, dequantize_counter); - } + Graph* graph) { + GET_IR_NODE_FROM_SUBGRAPH(op, op, Bloat16Ops); + + Quantizer quantizer(graph, op); + quantizer.AddQuantOps(); + quantize_counter += quantizer.get_counter(); + + DeQuantizer dequantizer(graph, op); + dequantizer.AddQuantOps(); + dequantize_counter += dequantizer.get_counter(); }; gpd(graph, handler); -} -void CPUBFloat16Pass::SetOutputDataType(ir::Graph* graph) const { - int dequantize_counter = 0; - AddReoderAfterDuplicatedOutputs(graph, dequantize_counter); - AddReoderAfterSingleOutputs(graph, dequantize_counter); + PrettyLogDetail("--- added %d quantize ops before bfloat16 op", + quantize_counter); PrettyLogDetail("--- added %d dequantize ops after bfloat16 op", dequantize_counter); } -void CPUBFloat16Pass::ApplyImpl(ir::Graph* graph) const { - SetInputDataType(graph); - SetOutputDataType(graph); -} - } // namespace ir } // namespace framework } // namespace paddle diff --git a/paddle/fluid/framework/ir/mkldnn/cpu_bfloat16_pass.h b/paddle/fluid/framework/ir/mkldnn/cpu_bfloat16_pass.h index 3a7271f7ddc59..69c7ce35162ff 100644 --- a/paddle/fluid/framework/ir/mkldnn/cpu_bfloat16_pass.h +++ b/paddle/fluid/framework/ir/mkldnn/cpu_bfloat16_pass.h @@ -24,8 +24,6 @@ namespace ir { class CPUBFloat16Pass : public Pass { protected: - void SetInputDataType(ir::Graph* graph) const; - void SetOutputDataType(ir::Graph* graph) const; void ApplyImpl(ir::Graph* graph) const override; }; diff --git a/paddle/fluid/framework/ir/mkldnn/cpu_bfloat16_placement_pass.cc b/paddle/fluid/framework/ir/mkldnn/cpu_bfloat16_placement_pass.cc index d89891ec3c857..fc7a53c4e7923 100644 --- a/paddle/fluid/framework/ir/mkldnn/cpu_bfloat16_placement_pass.cc +++ b/paddle/fluid/framework/ir/mkldnn/cpu_bfloat16_placement_pass.cc @@ -27,8 +27,16 @@ namespace ir { using string::PrettyLogDetail; -void CPUBfloat16PlacementPass::SetMkldnnDataType( - ir::Graph* graph, int* bfloat16_operators) const { +void CPUBfloat16PlacementPass::ApplyImpl(ir::Graph* graph) const { + int bfloat16_operators = 0; + bfloat16_operators += SetMkldnnDataType(graph); + bfloat16_operators -= RemoveOrphanedOperators(graph); + bfloat16_operators -= RemoveUnsupportedOperators(graph); + PrettyLogDetail("--- marked %d operators to bfloat16 ", + bfloat16_operators); +} + +int CPUBfloat16PlacementPass::SetMkldnnDataType(ir::Graph* graph) const { const auto& op_types_list = Get>("bfloat16_enabled_op_types"); // set mkldnn_data_type to bfloat16 to all operators that are in @@ -39,6 +47,7 @@ void CPUBfloat16PlacementPass::SetMkldnnDataType( "bfloat16_placement"}; bfloat16_placement_pattern(op_types_list); + int detected_operators = 0; auto handler = [&](const GraphPatternDetector::subgraph_t& subgraph, Graph* g) { GET_IR_NODE_FROM_SUBGRAPH(op_in, op_in, bfloat16_placement_pattern); @@ -50,58 +59,58 @@ void CPUBfloat16PlacementPass::SetMkldnnDataType( if ((op->Op()->HasAttr("mkldnn_data_type") || op->Op()->HasProtoAttr("mkldnn_data_type")) && !platform::HasOpINT8DataType(op->Op())) { + VLOG(4) << "--- marked " << op->Op()->Type() + << " operator to bfloat16 "; op->Op()->SetAttr("mkldnn_data_type", std::string("bfloat16")); - (*bfloat16_operators)++; + detected_operators++; } }; gpd(graph, handler); + return detected_operators; } -void CPUBfloat16PlacementPass::RemoveOrphanedOperators( - ir::Graph* graph, int* bfloat16_operators) const { +int CPUBfloat16PlacementPass::RemoveOrphanedOperators(ir::Graph* graph) const { // find orphaned bfloat16 operator that is between two float32 operators // revert mkldnn_data_type attr to float32 GraphPatternDetector gpd; patterns::OrphanedBfloat16 orphaned_bfloat16_pattern{gpd.mutable_pattern(), "orphaned_bfloat16"}; orphaned_bfloat16_pattern(); + int detected_operators = 0; auto handler = [&](const GraphPatternDetector::subgraph_t& subgraph, Graph* g) { GET_IR_NODE_FROM_SUBGRAPH(op, op, orphaned_bfloat16_pattern); op->Op()->SetAttr("mkldnn_data_type", std::string("float32")); - bfloat16_operators--; + VLOG(4) << "--- demarked " << op->Op()->Type() << " operator to bfloat16 "; + detected_operators++; }; gpd(graph, handler); + return detected_operators; } -void CPUBfloat16PlacementPass::RemoveUnsupportedOperators( - ir::Graph* graph, int* bfloat16_operators) const { +int CPUBfloat16PlacementPass::RemoveUnsupportedOperators( + ir::Graph* graph) const { // now quantize is supported FP32 only, so try to find // bfloat16 operator that input type is not FP32 GraphPatternDetector gpd; patterns::UnsupportedBfloat16 unsupported_bfloat16_pattern{ gpd.mutable_pattern(), "unsupported_bfloat16"}; unsupported_bfloat16_pattern(); + int detected_operators = 0; auto handler = [&](const GraphPatternDetector::subgraph_t& subgraph, Graph* g) { GET_IR_NODE_FROM_SUBGRAPH(prev_out, prev_out, unsupported_bfloat16_pattern); GET_IR_NODE_FROM_SUBGRAPH(op, op, unsupported_bfloat16_pattern); if ((prev_out->Var()->GetDataType() != proto::VarType::FP32)) { op->Op()->SetAttr("mkldnn_data_type", std::string("float32")); - bfloat16_operators--; + VLOG(4) << "--- demarked " << op->Op()->Type() + << " operator to bfloat16 "; + detected_operators++; } }; gpd(graph, handler); -} - -void CPUBfloat16PlacementPass::ApplyImpl(ir::Graph* graph) const { - int bfloat16_operators = 0; - SetMkldnnDataType(graph, &bfloat16_operators); - RemoveOrphanedOperators(graph, &bfloat16_operators); - RemoveUnsupportedOperators(graph, &bfloat16_operators); - PrettyLogDetail("--- marked %d operators to bfloat16 ", - bfloat16_operators); + return detected_operators; } } // namespace ir diff --git a/paddle/fluid/framework/ir/mkldnn/cpu_bfloat16_placement_pass.h b/paddle/fluid/framework/ir/mkldnn/cpu_bfloat16_placement_pass.h index facc4c4c55221..63848298a879a 100644 --- a/paddle/fluid/framework/ir/mkldnn/cpu_bfloat16_placement_pass.h +++ b/paddle/fluid/framework/ir/mkldnn/cpu_bfloat16_placement_pass.h @@ -26,14 +26,11 @@ namespace ir { */ class CPUBfloat16PlacementPass : public Pass { protected: - void SetMkldnnDataType(ir::Graph* graph, int* bfloat16_operators) const; - - void RemoveOrphanedOperators(ir::Graph* graph, int* bfloat16_operators) const; - - void RemoveUnsupportedOperators(ir::Graph* graph, - int* bfloat16_operators) const; - void ApplyImpl(ir::Graph* graph) const override; + + int SetMkldnnDataType(ir::Graph* graph) const; + int RemoveOrphanedOperators(ir::Graph* graph) const; + int RemoveUnsupportedOperators(ir::Graph* graph) const; }; } // namespace ir diff --git a/paddle/fluid/framework/ir/mkldnn/shuffle_channel_mkldnn_detect_pass_tester.cc b/paddle/fluid/framework/ir/mkldnn/shuffle_channel_mkldnn_detect_pass_tester.cc new file mode 100644 index 0000000000000..fe42e8f96f851 --- /dev/null +++ b/paddle/fluid/framework/ir/mkldnn/shuffle_channel_mkldnn_detect_pass_tester.cc @@ -0,0 +1,83 @@ +// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include +#include + +#include "paddle/fluid/framework/ir/mkldnn/shuffle_channel_mkldnn_detect_pass.h" +#include "paddle/fluid/framework/ir/pass_tester_helper.h" + +namespace paddle { +namespace framework { +namespace ir { + +void AddVarToScope(Scope* param_scope, const std::string& name, + const DDim& dims) { + auto* tensor = param_scope->Var(name)->GetMutable(); + tensor->Resize(dims); + tensor->mutable_data(platform::CPUPlace()); +} + +Scope* CreateParamScope() { + auto param_scope = new Scope(); + AddVarToScope(param_scope, "prog_x", {1, 128, 52, 52}); + return param_scope; +} + +void MainTest() { + Layers layers; + auto prog_x = layers.data("prog_x", {1, 128, 52, 52}); + auto first_reshape2 = layers.reshape2(prog_x, {-1, 2, 64, 52, 52}, true); + first_reshape2->SetShape({-1, 2, 64, 52, 52}); + auto transpose2 = layers.transpose2(first_reshape2, {0, 2, 1, 3, 4}, true); + transpose2->SetShape({-1, 64, 2, 52, 52}); + auto second_reshape2 = layers.reshape2(transpose2, {-1, 128, 52, 52}, true); + second_reshape2->SetShape({-1, 128, 52, 52}); + + std::unique_ptr graph(new ir::Graph(layers.main_program())); + graph->Set("__param_scope__", CreateParamScope()); + + int added_nodes = 1; // shuffle_channel + int removed_nodes = 5; // 2 * reshape, reshape_out, transpose, transpose_out + + int original_nodes_num = graph->Nodes().size(); + auto pass = + PassRegistry::Instance().Get("shuffle_channel_mkldnn_detect_pass"); + graph.reset(pass->Apply(graph.release())); + int current_nodes_num = graph->Nodes().size(); + + EXPECT_EQ(current_nodes_num, + original_nodes_num + added_nodes - removed_nodes); + EXPECT_EQ(GetNumOpNodes(graph, "reshape2"), 0); + EXPECT_EQ(GetNumOpNodes(graph, "transpose2"), 0); + EXPECT_EQ(GetNumOpNodes(graph, "shuffle_channel"), 1); + + for (const auto* node : graph->Nodes()) { + if (node->IsOp() && node->Op()->Type() == "shuffle_channel") { + const auto* op = node->Op(); + ASSERT_TRUE(op->HasAttr("use_mkldnn")); + EXPECT_TRUE(BOOST_GET_CONST(bool, op->GetAttr("use_mkldnn"))); + } + } +} + +TEST(ShuffleChannelOneDNNDetectPass, ShuffleChannelOneDNNDetectPassTest) { + MainTest(); +} + +} // namespace ir +} // namespace framework +} // namespace paddle + +USE_PASS(shuffle_channel_mkldnn_detect_pass); diff --git a/paddle/fluid/framework/naive_executor.cc b/paddle/fluid/framework/naive_executor.cc index f30d1ea1b83dd..dba3b3ff1e690 100644 --- a/paddle/fluid/framework/naive_executor.cc +++ b/paddle/fluid/framework/naive_executor.cc @@ -147,11 +147,16 @@ void NaiveExecutor::ResetTrtOps(int num) { int engine_predictor_id = trtop->Attr("predictor_id"); std::string engine_name = engine_key + std::to_string(engine_predictor_id); - operators::TensorRTEngine *trt_engine = - paddle::inference::Singleton< + operators::TensorRTEngine *trt_engine = nullptr; + // can't get trt engine if int8 calibration table data process. + if (paddle::inference::Singleton< inference::tensorrt::TRTEngineManager>::Global() - .Get(engine_name); - if (trt_engine->with_dynamic_shape()) { + .Has(engine_name)) { + trt_engine = paddle::inference::Singleton< + inference::tensorrt::TRTEngineManager>::Global() + .Get(engine_name); + } + if (trt_engine && trt_engine->with_dynamic_shape()) { LOG(INFO) << "rebuild trt engine, this may cost a lot of time!"; trt_engine->ResetContext(); trt_engine->ClearTensorMap(); diff --git a/paddle/fluid/framework/new_executor/interpretercore.cc b/paddle/fluid/framework/new_executor/interpretercore.cc index a4fcf0773f623..6735406aacde7 100644 --- a/paddle/fluid/framework/new_executor/interpretercore.cc +++ b/paddle/fluid/framework/new_executor/interpretercore.cc @@ -22,6 +22,7 @@ #include "paddle/fluid/framework/operator.h" #include "paddle/fluid/platform/os_info.h" #include "paddle/fluid/platform/profiler/event_tracing.h" +#include "paddle/phi/core/kernel_context.h" #ifdef PADDLE_WITH_MKLDNN #include "paddle/fluid/platform/mkldnn_helper.h" #endif diff --git a/paddle/fluid/framework/new_executor/interpretercore_util.cc b/paddle/fluid/framework/new_executor/interpretercore_util.cc index afddcb580b9d8..d6de37a72c772 100644 --- a/paddle/fluid/framework/new_executor/interpretercore_util.cc +++ b/paddle/fluid/framework/new_executor/interpretercore_util.cc @@ -19,6 +19,7 @@ #include "paddle/fluid/operators/controlflow/conditional_block_op_helper.h" #include "paddle/fluid/operators/controlflow/recurrent_op_helper.h" #include "paddle/fluid/operators/controlflow/while_op_helper.h" +#include "paddle/phi/core/kernel_context.h" #include "paddle/phi/core/kernel_factory.h" #ifdef PADDLE_WITH_MKLDNN @@ -740,7 +741,7 @@ std::map> get_downstream_map( VLOG(6) << "downstream count: " << downstream_map_count(); VLOG(6) << "downstream_map: " << std::endl << downstream_map_to_str(); - return std::move(downstream); + return downstream; } std::map> build_op_downstream_map( @@ -994,7 +995,7 @@ std::map> build_op_downstream_map( std::ostream_iterator(oss, " ")); VLOG(10) << oss.str(); } - return std::move(get_downstream_map(op2dependences, op_happens_before)); + return get_downstream_map(op2dependences, op_happens_before); } } // namespace interpreter diff --git a/paddle/fluid/framework/new_executor/new_executor_defs.cc b/paddle/fluid/framework/new_executor/new_executor_defs.cc index 3c2395d4320a1..c75a7871d63e9 100644 --- a/paddle/fluid/framework/new_executor/new_executor_defs.cc +++ b/paddle/fluid/framework/new_executor/new_executor_defs.cc @@ -328,21 +328,21 @@ bool InterpretercoreInferShapeContext::IsRunMKLDNNKernel() const { } // TODO(paddle-dev): Can this be template? -paddle::SmallVector +paddle::small_vector InterpretercoreInferShapeContext::GetInputVarPtrs( const std::string& name) const { const std::vector& vars = InputVars(name); - paddle::SmallVector res; + paddle::small_vector res; res.reserve(vars.size()); res.insert(res.begin(), vars.begin(), vars.end()); return res; } -paddle::SmallVector +paddle::small_vector InterpretercoreInferShapeContext::GetOutputVarPtrs( const std::string& name) const { const std::vector& vars = OutputVars(name); - paddle::SmallVector res; + paddle::small_vector res; res.reserve(vars.size()); res.insert(res.begin(), vars.begin(), vars.end()); return res; @@ -365,6 +365,11 @@ std::vector InterpretercoreInferShapeContext::GetInputsDim( return GetDims(vars); } +proto::VarType::Type InterpretercoreInferShapeContext::GetInputVarType( + const std::string& name) const { + return GetVarType(InputVars(name).at(0)); +} + std::vector InterpretercoreInferShapeContext::GetInputsVarType( const std::string& name) const { @@ -393,6 +398,16 @@ void InterpretercoreInferShapeContext::SetOutputsDim( SetDims(vars, dims); } +const phi::ArgumentMappingFn* +InterpretercoreInferShapeContext::GetPhiArgumentMappingFn() const { + return phi::OpUtilsMap::Instance().GetArgumentMappingFn(op_.Type()); +} + +const phi::KernelSignature* +InterpretercoreInferShapeContext::GetPhiDefaultKernelSignature() const { + return &phi::DefaultKernelSignatureMap::Instance().Get(op_.Type()); +} + void InterpretercoreInferShapeContext::SetSkipLoD(bool skip) { can_skip_lod_ = skip; } diff --git a/paddle/fluid/framework/new_executor/new_executor_defs.h b/paddle/fluid/framework/new_executor/new_executor_defs.h index 28b9f6f0130f5..20e51145a51b2 100644 --- a/paddle/fluid/framework/new_executor/new_executor_defs.h +++ b/paddle/fluid/framework/new_executor/new_executor_defs.h @@ -90,16 +90,18 @@ class InterpretercoreInferShapeContext : public InferShapeContext { bool IsRunMKLDNNKernel() const override; // TODO(paddle-dev): Can this be template? - paddle::SmallVector + paddle::small_vector GetInputVarPtrs(const std::string& name) const override; - paddle::SmallVector + paddle::small_vector GetOutputVarPtrs(const std::string& name) const override; DDim GetInputDim(const std::string& name) const override; std::vector GetInputsDim(const std::string& name) const override; + proto::VarType::Type GetInputVarType(const std::string& name) const override; + std::vector GetInputsVarType( const std::string& name) const override; @@ -111,6 +113,10 @@ class InterpretercoreInferShapeContext : public InferShapeContext { void SetOutputsDim(const std::string& name, const std::vector& dims) override; + const phi::ArgumentMappingFn* GetPhiArgumentMappingFn() const override; + + const phi::KernelSignature* GetPhiDefaultKernelSignature() const override; + void SetSkipLoD(bool skip); protected: diff --git a/paddle/fluid/framework/op_desc.cc b/paddle/fluid/framework/op_desc.cc index d27bf0e150f97..87d3a048d0be0 100644 --- a/paddle/fluid/framework/op_desc.cc +++ b/paddle/fluid/framework/op_desc.cc @@ -202,10 +202,10 @@ class CompileTimeInferShapeContext : public InferShapeContext { } } - paddle::SmallVector + paddle::small_vector GetInputVarPtrs(const std::string &name) const override { const std::vector arg_names = Inputs(name); - paddle::SmallVector res; + paddle::small_vector res; res.reserve(arg_names.size()); std::transform(arg_names.begin(), arg_names.end(), std::back_inserter(res), [this](const std::string &name) { @@ -214,10 +214,10 @@ class CompileTimeInferShapeContext : public InferShapeContext { return res; } - paddle::SmallVector + paddle::small_vector GetOutputVarPtrs(const std::string &name) const override { const std::vector arg_names = Outputs(name); - paddle::SmallVector res; + paddle::small_vector res; res.reserve(arg_names.size()); std::transform(arg_names.begin(), arg_names.end(), std::back_inserter(res), [this](const std::string &name) { @@ -245,6 +245,10 @@ class CompileTimeInferShapeContext : public InferShapeContext { bool IsRunMKLDNNKernel() const override; + proto::VarType::Type GetInputVarType(const std::string &name) const override { + return GetVarType(Inputs(name).at(0)); + } + std::vector GetInputsVarType( const std::string &name) const override { return GetVarTypes(Inputs(name)); @@ -271,6 +275,14 @@ class CompileTimeInferShapeContext : public InferShapeContext { SetDims(names, dims); } + const phi::ArgumentMappingFn *GetPhiArgumentMappingFn() const override { + return phi::OpUtilsMap::Instance().GetArgumentMappingFn(op_.Type()); + } + + const phi::KernelSignature *GetPhiDefaultKernelSignature() const override { + return &phi::DefaultKernelSignatureMap::Instance().Get(op_.Type()); + } + protected: std::vector GetVarTypes( const std::vector &names) const { diff --git a/paddle/fluid/framework/op_registry.cc b/paddle/fluid/framework/op_registry.cc index d69edef7840f5..d14254b7355c9 100644 --- a/paddle/fluid/framework/op_registry.cc +++ b/paddle/fluid/framework/op_registry.cc @@ -21,13 +21,17 @@ namespace framework { std::unique_ptr OpRegistry::CreateOp( const std::string& type, const VariableNameMap& inputs, - const VariableNameMap& outputs, AttributeMap attrs, bool attr_check) { + const VariableNameMap& outputs, const AttributeMap& attrs, + bool attr_check) { auto& info = OpInfoMap::Instance().Get(type); if (attr_check && info.Checker() != nullptr) { - info.Checker()->Check(&attrs); + auto tmp_attrs = attrs; + info.Checker()->Check(&tmp_attrs); + return std::unique_ptr( + info.Creator()(type, inputs, outputs, tmp_attrs)); } - auto op = info.Creator()(type, inputs, outputs, attrs); - return std::unique_ptr(op); + return std::unique_ptr( + info.Creator()(type, inputs, outputs, attrs)); } static VariableNameMap ConvertOpDescVarsToVarNameMap( diff --git a/paddle/fluid/framework/op_registry.h b/paddle/fluid/framework/op_registry.h index eb40a49b4066a..a1f07f9f2520e 100644 --- a/paddle/fluid/framework/op_registry.h +++ b/paddle/fluid/framework/op_registry.h @@ -129,7 +129,7 @@ class OpRegistry { static std::unique_ptr CreateOp(const std::string& type, const VariableNameMap& inputs, const VariableNameMap& outputs, - AttributeMap attrs, + const AttributeMap& attrs, bool attr_check = true); static std::unique_ptr CreateOp(const proto::OpDesc& op_desc); diff --git a/paddle/fluid/framework/operator.cc b/paddle/fluid/framework/operator.cc index 871c459c71764..0c22321996b8f 100644 --- a/paddle/fluid/framework/operator.cc +++ b/paddle/fluid/framework/operator.cc @@ -1,11 +1,8 @@ /* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved. - Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at - http://www.apache.org/licenses/LICENSE-2.0 - Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -35,6 +32,7 @@ limitations under the License. */ #include "paddle/fluid/platform/profiler/event_tracing.h" #include "paddle/phi/common/int_array.h" #include "paddle/phi/common/scalar.h" +#include "paddle/phi/core/kernel_context.h" #include "paddle/phi/core/kernel_factory.h" #include "paddle/phi/ops/compat/signatures.h" @@ -939,25 +937,25 @@ class RuntimeInferShapeContext : public InferShapeContext { return ((op_with_kernel.kernel_type()) && (op_with_kernel.kernel_type()->data_layout_ == framework::DataLayout::kMKLDNN)); - } catch (std::bad_cast exp) { + } catch (const std::bad_cast& exp) { return false; } } // TODO(paddle-dev): Can this be template? - paddle::SmallVector + paddle::small_vector GetInputVarPtrs(const std::string& name) const override { const std::vector& vars = InputVars(name); - paddle::SmallVector res; + paddle::small_vector res; res.reserve(vars.size()); res.insert(res.begin(), vars.begin(), vars.end()); return res; } - paddle::SmallVector + paddle::small_vector GetOutputVarPtrs(const std::string& name) const override { const std::vector& vars = OutputVars(name); - paddle::SmallVector res; + paddle::small_vector res; res.reserve(vars.size()); res.insert(res.begin(), vars.begin(), vars.end()); return res; @@ -978,6 +976,10 @@ class RuntimeInferShapeContext : public InferShapeContext { return GetDims(vars); } + proto::VarType::Type GetInputVarType(const std::string& name) const override { + return GetVarType(InputVars(name).at(0)); + } + std::vector GetInputsVarType( const std::string& name) const override { return GetVarTypes(InputVars(name)); @@ -1004,6 +1006,14 @@ class RuntimeInferShapeContext : public InferShapeContext { SetDims(vars, dims); } + const phi::ArgumentMappingFn* GetPhiArgumentMappingFn() const override { + return phi::OpUtilsMap::Instance().GetArgumentMappingFn(op_.Type()); + } + + const phi::KernelSignature* GetPhiDefaultKernelSignature() const override { + return &phi::DefaultKernelSignatureMap::Instance().Get(op_.Type()); + } + protected: DDim GetDim(Variable* var) const { PADDLE_ENFORCE_NOT_NULL( @@ -1198,8 +1208,10 @@ bool OperatorWithKernel::SupportsMKLDNN( bool OperatorWithKernel::CanMKLDNNBeUsed(const framework::ExecutionContext& ctx, proto::VarType::Type data_type) const { - bool use_mkldnn_ctx = ctx.HasAttr("use_mkldnn") && - ctx.Attr("use_mkldnn") && + const auto& attrs_map = ctx.Attrs(); + auto iter = attrs_map.find("use_mkldnn"); + bool use_mkldnn_ctx = iter != attrs_map.end() && + BOOST_GET_CONST(bool, iter->second) && platform::is_cpu_place(ctx.GetPlace()); return use_mkldnn_ctx && this->SupportsMKLDNN(data_type); } @@ -1266,6 +1278,12 @@ void OperatorWithKernel::RunImpl(const Scope& scope, dev_ctx = pool.Get(kernel_type_->place_); } +// TODO(Liu-xiandong): Now we are using too much if-else and hard code in XPU +// device, it's ugly, and we will refactor in the future. +#if defined(PADDLE_WITH_XPU_KP) + bool use_phi_xpu_kp = false; +#endif + // TODO(chenweihang): Now we are still reusing a lot of the original fluid // implementation, this is a gradual replacement process // TODO(chenweihang): in the first phase of project, we only support CPU, CUDA @@ -1274,16 +1292,55 @@ void OperatorWithKernel::RunImpl(const Scope& scope, phi::KernelKey pt_kernel_key; std::string pt_kernel_name; if (phi::KernelFactory::Instance().HasCompatiblePhiKernel(type_)) { - if (pt_kernel_signature_ == nullptr || pt_kernel_ == nullptr) { - pt_kernel_signature_.reset( - new KernelSignature(std::move(GetExpectedPhiKernelArgs(exe_ctx)))); - VLOG(6) << *pt_kernel_signature_.get(); + if (kernel_signature_ == nullptr || pt_kernel_ == nullptr) { + kernel_signature_.reset(new phi::KernelSignature( + std::move(GetExpectedPhiKernelArgs(exe_ctx)))); + VLOG(6) << *kernel_signature_.get(); kernel_type_.reset( new OpKernelType(std::move(InnerGetExpectedKernelType(exe_ctx)))); dev_ctx = pool.Get(kernel_type_->place_); - pt_kernel_name = pt_kernel_signature_->name; + pt_kernel_name = kernel_signature_->name; +// NOTE(Liu-xiandong): The register kernel used KP have library_type[KP], +// But the default library_type is Plain, so we need to modify the +// library_type here, otherwise it can't work. +#ifdef PADDLE_WITH_XPU_KP + if (paddle::platform::is_xpu_place(kernel_type_->place_)) { + bool use_xpu_kp_kernel_rt = + FLAGS_run_kp_kernel && + paddle::platform::is_xpu_kp_support_op(type_, *kernel_type_); + bool use_xpu_kp_kernel_debug = + paddle::platform::is_in_xpu_kpwhite_list(type_); + if (use_xpu_kp_kernel_rt) { + VLOG(3) << "phi xpu_kp using rt mode in static graph"; + } + if (use_xpu_kp_kernel_debug) { + VLOG(3) << "phi xpu_kp using debug mode in static graph"; + } + bool is_xpu_kp_support = + (use_xpu_kp_kernel_rt || use_xpu_kp_kernel_debug); + if (is_xpu_kp_support) { + auto expected_kernel_key_library_type = kernel_type_->library_type_; + kernel_type_->library_type_ = LibraryType::kKP; + VLOG(3) << "modifing XPU KP kernel in static graph: " + << pt_kernel_name + << ", using_kernel_key:" << *kernel_type_.get(); + auto try_pt_kernel_key = + TransOpKernelTypeToPhiKernelKey(*kernel_type_.get()); + if (!phi::KernelFactory::Instance().HasKernel(pt_kernel_name, + try_pt_kernel_key)) { + kernel_type_->library_type_ = expected_kernel_key_library_type; + VLOG(3) << "modify XPU KP kernel in static graph: " + << pt_kernel_name << " is failed " << *kernel_type_.get(); + } else { + use_phi_xpu_kp = true; + VLOG(3) << "modify XPU KP kernel in static graph: " + << pt_kernel_name << " is succeed " << *kernel_type_.get(); + } + } + } +#endif pt_kernel_key = TransOpKernelTypeToPhiKernelKey(*kernel_type_.get()); pt_kernel_.reset( new phi::Kernel(phi::KernelFactory::Instance().SelectKernel( @@ -1298,10 +1355,10 @@ void OperatorWithKernel::RunImpl(const Scope& scope, << "` not found."; } } else { - pt_kernel_name = pt_kernel_signature_->name; -// NOTE(Liu-xiandong): The register kernel used KP have library_type[KP], -// But the default library_type is Plain, so we need to modify the -// library_type here, otherwise it can't work. + pt_kernel_name = kernel_signature_->name; +// NOTE(Liu-xiandong):In my ctest, this branch do not be executed, +// I can't understand it, it's really confusing. +// But we still need to keep this to avoid errors. #ifdef PADDLE_WITH_XPU_KP if (paddle::platform::is_xpu_place(kernel_type_->place_)) { bool use_xpu_kp_kernel_rt = @@ -1320,15 +1377,20 @@ void OperatorWithKernel::RunImpl(const Scope& scope, if (is_xpu_kp_support) { auto expected_kernel_key_library_type = kernel_type_->library_type_; kernel_type_->library_type_ = LibraryType::kKP; - VLOG(3) << "modifing XPU KP kernel in static graph: " << type_ + VLOG(3) << "modifing XPU KP kernel in static graph: " + << pt_kernel_name << ", using_kernel_key:" << *kernel_type_.get(); auto try_pt_kernel_key = TransOpKernelTypeToPhiKernelKey(*kernel_type_.get()); if (!phi::KernelFactory::Instance().HasKernel(pt_kernel_name, try_pt_kernel_key)) { kernel_type_->library_type_ = expected_kernel_key_library_type; - VLOG(3) << "modify XPU KP kernel in static graph: " << type_ - << " is failed " << *kernel_type_.get(); + VLOG(3) << "modify XPU KP kernel in static graph: " + << pt_kernel_name << " is failed " << *kernel_type_.get(); + } else { + use_phi_xpu_kp = true; + VLOG(3) << "modify XPU KP kernel in static graph: " + << pt_kernel_name << " is succeed " << *kernel_type_.get(); } } } @@ -1345,11 +1407,25 @@ void OperatorWithKernel::RunImpl(const Scope& scope, !paddle::platform::is_xpu_support_op(type_, *kernel_type_.get()) || paddle::platform::is_in_xpu_black_list(type_); #endif +#ifdef PADDLE_WITH_XPU_KP + bool use_xpu_kp_kernel_rt = + paddle::platform::is_xpu_place(kernel_type_->place_) && + FLAGS_run_kp_kernel && + paddle::platform::is_xpu_kp_support_op(type_, *kernel_type_); + bool use_xpu_kp_kernel_debug = + paddle::platform::is_xpu_place(kernel_type_->place_) && + paddle::platform::is_in_xpu_kpwhite_list(type_); + bool is_xpu_kp_support = (use_xpu_kp_kernel_rt || use_xpu_kp_kernel_debug); +#endif + if (pt_kernel_->IsValid() #if defined(PADDLE_WITH_XPU) && !defined(PADDLE_WITH_XPU_KP) && !is_xpu_unsupport #endif - ) { +#if defined(PADDLE_WITH_XPU_KP) + && (!is_xpu_unsupport || use_phi_xpu_kp) +#endif + ) { run_phi_kernel_ = true; } else { auto& all_op_kernels = AllOpKernels(); @@ -1359,15 +1435,6 @@ void OperatorWithKernel::RunImpl(const Scope& scope, // we need to select the heterogeneous kernel in fluid, but the kernel // registered in KP use library_type[KP], we need to modify it. #ifdef PADDLE_WITH_XPU_KP - bool use_xpu_kp_kernel_rt = - paddle::platform::is_xpu_place(kernel_type_->place_) && - FLAGS_run_kp_kernel && - paddle::platform::is_xpu_kp_support_op(type_, *kernel_type_); - bool use_xpu_kp_kernel_debug = - paddle::platform::is_xpu_place(kernel_type_->place_) && - paddle::platform::is_in_xpu_kpwhite_list(type_); - bool is_xpu_kp_support = - (use_xpu_kp_kernel_rt || use_xpu_kp_kernel_debug); if (is_xpu_kp_support) { kernel_type_->library_type_ = LibraryType::kKP; } @@ -1444,8 +1511,7 @@ void OperatorWithKernel::RunImpl(const Scope& scope, phi::KernelContext pt_kernel_context; // Do data transform before building KernelContext // TODO(zhiqiu): support TransferInplaceVarsBack - PreparePhiData(exec_scope, *pt_kernel_, *pt_kernel_signature_, - runtime_ctx); + PreparePhiData(exec_scope, *pt_kernel_, *kernel_signature_, runtime_ctx); BuildPhiKernelContext(*runtime_ctx, dev_ctx, &pt_kernel_context); (*pt_kernel_)(&pt_kernel_context); } else { @@ -1540,14 +1606,14 @@ OpKernelType OperatorWithKernel::InnerGetExpectedKernelType( phi::KernelKey OperatorWithKernel::ChoosePhiKernel( const ExecutionContext& ctx) const { - pt_kernel_signature_.reset( - new KernelSignature(std::move(GetExpectedPhiKernelArgs(ctx)))); - VLOG(6) << *pt_kernel_signature_.get(); + kernel_signature_.reset( + new phi::KernelSignature(std::move(GetExpectedPhiKernelArgs(ctx)))); + VLOG(6) << *kernel_signature_.get(); kernel_type_.reset( new OpKernelType(std::move(InnerGetExpectedKernelType(ctx)))); - auto pt_kernel_name = pt_kernel_signature_->name; + auto pt_kernel_name = kernel_signature_->name; auto pt_kernel_key = TransOpKernelTypeToPhiKernelKey(*kernel_type_.get()); pt_kernel_.reset(new phi::Kernel(phi::KernelFactory::Instance().SelectKernel( pt_kernel_name, pt_kernel_key))); @@ -1595,7 +1661,7 @@ void OperatorWithKernel::ChooseKernel(const ExecutionContext& ctx) const { (kernel_iter == kernels.end() || !paddle::platform::is_xpu_support_op(type_, expected_kernel_key) || paddle::platform::is_in_xpu_black_list(type_))) { - VLOG(3) << "missing XPU kernel: " << type_ + VLOG(3) << "fluid missing XPU kernel: " << type_ << ", expected_kernel_key:" << expected_kernel_key << ", fallbacking to CPU one!"; expected_kernel_key.place_ = platform::CPUPlace(); @@ -1611,10 +1677,10 @@ void OperatorWithKernel::ChooseKernel(const ExecutionContext& ctx) const { bool use_xpu_kp_kernel_debug = paddle::platform::is_in_xpu_kpwhite_list(type_); if (use_xpu_kp_kernel_rt) { - VLOG(3) << "xpu_kp using rt mode "; + VLOG(3) << "fluid xpu_kp using rt mode "; } if (use_xpu_kp_kernel_debug) { - VLOG(3) << "xpu_kp using debug mode "; + VLOG(3) << "fluid xpu_kp using debug mode "; } bool is_xpu_kp_support = (use_xpu_kp_kernel_rt || use_xpu_kp_kernel_debug); if (is_xpu_kp_support) { @@ -1631,7 +1697,7 @@ void OperatorWithKernel::ChooseKernel(const ExecutionContext& ctx) const { expected_kernel_key.place_ = platform::CPUPlace(); kernel_iter = kernels.find(expected_kernel_key); } else { - VLOG(3) << "using XPU KP kernel: " << type_ + VLOG(3) << "fluid using XPU KP kernel: " << type_ << ", using_kernel_key:" << expected_kernel_key; } } @@ -1640,7 +1706,7 @@ void OperatorWithKernel::ChooseKernel(const ExecutionContext& ctx) const { paddle::platform::is_in_xpu_black_list(type_)); if (!is_xpu_kp_support && (kernel_iter == kernels.end() || is_xpu_unsupport)) { - VLOG(3) << "missing XPU kernel: " << type_ + VLOG(3) << "fluid missing XPU kernel: " << type_ << ", expected_kernel_key:" << expected_kernel_key << ", fallbacking to CPU one!"; expected_kernel_key.place_ = platform::CPUPlace(); @@ -1962,6 +2028,36 @@ Scope* OperatorWithKernel::PrepareData( } void OperatorWithKernel::ParseInputDataType( + const Variable* var, const std::string& name, + proto::VarType::Type* data_type) const { + if (var != nullptr) { + const Tensor* t = nullptr; + if (var->IsType()) { + t = &var->Get(); + } else if (var->IsType()) { + t = &var->Get(); + } else if (var->IsType()) { + t = &(var->Get().value()); + } else if (var->IsType()) { + auto t_arr = &var->Get(); + for (size_t j = 0; j < t_arr->size(); j++) { + if (t_arr->at(j).IsInitialized()) { + t = &(t_arr->at(j)); + } + } + } + if (t != nullptr) { + PADDLE_ENFORCE_EQ( + t->IsInitialized(), true, + platform::errors::InvalidArgument("The %s Op's Input Variable `%s` " + "contains uninitialized Tensor.", + Type(), name)); + *data_type = paddle::framework::TransToProtoVarType(t->dtype()); + } + } +} + +void OperatorWithKernel::ParseMultiInputDataType( const std::vector& vars, const std::string& name, proto::VarType::Type* data_type) const { proto::VarType::Type default_data_type = @@ -2012,9 +2108,12 @@ proto::VarType::Type OperatorWithKernel::IndicateDataType( proto::VarType::Type dafault_data_type = static_cast(-1); proto::VarType::Type data_type = dafault_data_type; - for (auto& input : ctx.InNameList()) { - const std::vector vars = ctx.MultiInputVar(input); - ParseInputDataType(vars, input, &data_type); + for (auto* name : ctx.InNameList()) { + if (ctx.InputSize(*name) == 1UL) { + ParseInputDataType(ctx.InputVar(*name), *name, &data_type); + } else { + ParseMultiInputDataType(ctx.MultiInputVar(*name), *name, &data_type); + } } PADDLE_ENFORCE_NE( data_type, dafault_data_type, @@ -2028,7 +2127,11 @@ proto::VarType::Type OperatorWithKernel::IndicateVarDataType( proto::VarType::Type dafault_data_type = static_cast(-1); proto::VarType::Type data_type = dafault_data_type; - ParseInputDataType(ctx.MultiInputVar(name), name, &data_type); + if (ctx.InputSize(name) == 1UL) { + ParseInputDataType(ctx.InputVar(name), name, &data_type); + } else { + ParseMultiInputDataType(ctx.MultiInputVar(name), name, &data_type); + } PADDLE_ENFORCE_NE( data_type, dafault_data_type, platform::errors::InvalidArgument( @@ -2111,20 +2214,29 @@ OpKernelType OperatorWithKernel::GetKernelTypeForVar( tensor.layout()); } -KernelSignature OperatorWithKernel::GetExpectedPhiKernelArgs( +phi::KernelSignature OperatorWithKernel::GetExpectedPhiKernelArgs( const ExecutionContext& ctx) const { ExecutionArgumentMappingContext arg_mapping_ctx(ctx); if (arg_map_fn_ == nullptr) { - arg_map_fn_.reset(new phi::ArgumentMappingFn( - phi::OpUtilsMap::Instance().GetArgumentMappingFn(Type()))); + auto* arg_map_fn = phi::OpUtilsMap::Instance().GetArgumentMappingFn(type_); + if (arg_map_fn) { + arg_map_fn_.reset(new phi::ArgumentMappingFn(*arg_map_fn)); + } else { + auto func = [this]( + const phi::ArgumentMappingContext& ctx) -> phi::KernelSignature { + return phi::DefaultKernelSignatureMap::Instance().Get(type_); + }; + arg_map_fn_.reset(new phi::ArgumentMappingFn(func)); + } } return (*arg_map_fn_)(arg_mapping_ctx); } Scope* OperatorWithKernel::PreparePhiData( const Scope& scope, const phi::Kernel& pt_kernel, - const KernelSignature& pt_kernel_signature, RuntimeContext* ctx) const { - auto& input_names = std::get<0>(pt_kernel_signature.args); + const phi::KernelSignature& pt_kernel_signature, + RuntimeContext* ctx) const { + const auto& input_names = pt_kernel_signature.input_names; auto input_defs = pt_kernel.args_def().input_defs(); PADDLE_ENFORCE_EQ(input_names.size(), input_defs.size(), platform::errors::InvalidArgument( @@ -2176,11 +2288,15 @@ Scope* OperatorWithKernel::PreparePhiData( if (in_def.backend == phi::Backend::ALL_BACKEND) { continue; } - auto expected_place = phi::TransToPhiPlace(in_def.backend); - if (platform::is_same_place(tensor_in->place(), expected_place)) { + + auto tensor_backend = phi::TransToPhiBackend(tensor_in->place()); + if (in_def.backend == tensor_backend || + (in_def.backend == phi::Backend::GPUDNN && + tensor_backend == phi::Backend::GPU)) { continue; } + auto expected_place = phi::TransToPhiPlace(in_def.backend); VLOG(3) << "phi Transform Variable " << input_names[i] << " from " << tensor_in->place() << " to " << expected_place; @@ -2217,9 +2333,9 @@ void OperatorWithKernel::BuildPhiKernelContext( phi::KernelContext* pt_kernel_context) const { pt_kernel_context->SetDeviceContext(dev_ctx); - auto& input_names = std::get<0>(pt_kernel_signature_->args); - auto& attr_names = std::get<1>(pt_kernel_signature_->args); - auto& output_names = std::get<2>(pt_kernel_signature_->args); + auto& input_names = kernel_signature_->input_names; + auto& attr_names = kernel_signature_->attr_names; + auto& output_names = kernel_signature_->output_names; auto input_defs = pt_kernel_->args_def().input_defs(); auto attr_defs = pt_kernel_->args_def().attribute_defs(); @@ -2280,7 +2396,7 @@ void OperatorWithKernel::BuildPhiKernelContext( tensor_in = &(var->Get()); pt_kernel_context->EmplaceBackInputWithoutSetRange(tensor_in); } else if (var->IsType()) { - paddle::SmallVector tensor_vector; + paddle::small_vector tensor_vector; auto& tensor_array = var->Get(); for (auto& t : tensor_array) { tensor_vector.emplace_back(&t); @@ -2329,7 +2445,7 @@ void OperatorWithKernel::BuildPhiKernelContext( tensor_out = var->template GetMutable(); pt_kernel_context->EmplaceBackOutputWithoutSetRange(tensor_out); } else if (var->template IsType()) { - paddle::SmallVector tensor_vector; + paddle::small_vector tensor_vector; auto* tensor_array = var->template GetMutable(); // Note: If the input LoDTensorArray size is 0, the output @@ -2353,21 +2469,19 @@ void OperatorWithKernel::BuildPhiKernelContext( VLOG(4) << "Done outputs"; for (size_t i = 0; i < attr_names.size(); ++i) { - if (attr_defs[i].type_index == std::type_index(typeid(phi::IntArray))) { + if (attr_defs[i].type_index == phi::AttributeType::INT_ARRAY) { auto attr_iter = Attrs().find(attr_names[i]); if (attr_iter != Attrs().end()) { // shape is in the attribute - if (std::type_index(attr_iter->second.type()) == - std::type_index(typeid(std::vector))) { - pt_kernel_context->EmplaceBackAttr(std::move(phi::IntArray( - BOOST_GET_CONST(std::vector, attr_iter->second)))); - } else if (std::type_index(attr_iter->second.type()) == - std::type_index(typeid(std::vector))) { - pt_kernel_context->EmplaceBackAttr(std::move(phi::IntArray( - BOOST_GET_CONST(std::vector, attr_iter->second)))); - } else if (std::type_index(attr_iter->second.type()) == - std::type_index(typeid(int32_t))) { + auto& attr = attr_iter->second; + if (AttrTypeID(attr) == proto::AttrType::LONGS) { pt_kernel_context->EmplaceBackAttr(std::move( - phi::IntArray(&BOOST_GET_CONST(int32_t, attr_iter->second), 1))); + phi::IntArray(BOOST_GET_CONST(std::vector, attr)))); + } else if (AttrTypeID(attr) == proto::AttrType::INTS) { + pt_kernel_context->EmplaceBackAttr(std::move( + phi::IntArray(BOOST_GET_CONST(std::vector, attr)))); + } else if (AttrTypeID(attr) == proto::AttrType::INT) { + pt_kernel_context->EmplaceBackAttr( + std::move(phi::IntArray(&BOOST_GET_CONST(int32_t, attr), 1))); } else { PADDLE_THROW(platform::errors::Unimplemented( "Unsupported cast op attribute `%s` to IntArray when " @@ -2384,23 +2498,17 @@ void OperatorWithKernel::BuildPhiKernelContext( std::move(experimental::MakePhiIntArrayFromVarList(ins_vector))); } } - } else if (attr_defs[i].type_index == - std::type_index(typeid(phi::Scalar))) { - // TODO(chenweihang): support other attrs later - // TODO(zhangyunfei): Scalar should hold scaler type, and we should check - // attribtue type by attr_defs + } else if (attr_defs[i].type_index == phi::AttributeType::SCALAR) { auto attr_iter = Attrs().find(attr_names[i]); if (attr_iter != Attrs().end()) { // scalar is in the attribute - auto& attr = Attrs().at(attr_names[i]); - if (std::type_index(attr.type()) == std::type_index(typeid(float))) { + auto& attr = attr_iter->second; + if (AttrTypeID(attr) == proto::AttrType::FLOAT) { pt_kernel_context->EmplaceBackAttr( std::move(phi::Scalar(BOOST_GET_CONST(float, attr)))); - } else if (std::type_index(attr.type()) == - std::type_index(typeid(std::string))) { + } else if (AttrTypeID(attr) == proto::AttrType::STRING) { pt_kernel_context->EmplaceBackAttr( std::move(phi::Scalar(BOOST_GET_CONST(std::string, attr)))); - } else if (std::type_index(attr.type()) == - std::type_index(typeid(int))) { + } else if (AttrTypeID(attr) == proto::AttrType::INT) { pt_kernel_context->EmplaceBackAttr( std::move(phi::Scalar(BOOST_GET_CONST(int, attr)))); } else { @@ -2415,11 +2523,9 @@ void OperatorWithKernel::BuildPhiKernelContext( std::move(experimental::MakePhiScalarFromVar(*ins_vector.front()))); } - } else if (attr_defs[i].type_index == - std::type_index(typeid(std::vector))) { + } else if (attr_defs[i].type_index == phi::AttributeType::SCALARS) { auto& attr = Attrs().at(attr_names[i]); - if (std::type_index(attr.type()) == - std::type_index(typeid(std::vector))) { + if (AttrTypeID(attr) == proto::AttrType::INTS) { const auto& vec = BOOST_GET_CONST(std::vector, attr); std::vector scalar_list; scalar_list.reserve(vec.size()); @@ -2427,8 +2533,7 @@ void OperatorWithKernel::BuildPhiKernelContext( scalar_list.emplace_back(val); } pt_kernel_context->EmplaceBackAttr(std::move(scalar_list)); - } else if (std::type_index(attr.type()) == - std::type_index(typeid(std::vector))) { + } else if (AttrTypeID(attr) == proto::AttrType::LONGS) { const auto& vec = BOOST_GET_CONST(std::vector, attr); std::vector scalar_list; scalar_list.reserve(vec.size()); @@ -2436,8 +2541,7 @@ void OperatorWithKernel::BuildPhiKernelContext( scalar_list.emplace_back(val); } pt_kernel_context->EmplaceBackAttr(std::move(scalar_list)); - } else if (std::type_index(attr.type()) == - std::type_index(typeid(std::vector))) { + } else if (AttrTypeID(attr) == proto::AttrType::FLOATS) { const auto& vec = BOOST_GET_CONST(std::vector, attr); std::vector scalar_list; scalar_list.reserve(vec.size()); @@ -2445,8 +2549,7 @@ void OperatorWithKernel::BuildPhiKernelContext( scalar_list.emplace_back(val); } pt_kernel_context->EmplaceBackAttr(std::move(scalar_list)); - } else if (std::type_index(attr.type()) == - std::type_index(typeid(std::vector))) { + } else if (AttrTypeID(attr) == proto::AttrType::FLOAT64S) { const auto& vec = BOOST_GET_CONST(std::vector, attr); std::vector scalar_list; scalar_list.reserve(vec.size()); @@ -2461,9 +2564,8 @@ void OperatorWithKernel::BuildPhiKernelContext( attr_names[i])); } } else { - // TODO(chenweihang): support other attrs later auto attr_it = attrs_.find(attr_names[i]); - if (attr_defs[i].type_index == std::type_index(typeid(int))) { + if (attr_defs[i].type_index == phi::AttributeType::INT32) { if (attr_it == attrs_.end()) { auto in_it = ctx.inputs.find(attr_names[i]); if (in_it != ctx.inputs.end()) { @@ -2480,33 +2582,28 @@ void OperatorWithKernel::BuildPhiKernelContext( pt_kernel_context->EmplaceBackAttr( BOOST_GET_CONST(int, attr_it->second)); } - } else if (attr_defs[i].type_index == std::type_index(typeid(float))) { + } else if (attr_defs[i].type_index == phi::AttributeType::FLOAT32) { pt_kernel_context->EmplaceBackAttr( BOOST_GET_CONST(float, attr_it->second)); - } else if (attr_defs[i].type_index == std::type_index(typeid(bool))) { + } else if (attr_defs[i].type_index == phi::AttributeType::BOOL) { pt_kernel_context->EmplaceBackAttr( BOOST_GET_CONST(bool, attr_it->second)); - } else if (attr_defs[i].type_index == std::type_index(typeid(int64_t))) { + } else if (attr_defs[i].type_index == phi::AttributeType::INT64) { pt_kernel_context->EmplaceBackAttr( BOOST_GET_CONST(int64_t, attr_it->second)); - } else if (attr_defs[i].type_index == - std::type_index(typeid(std::string))) { + } else if (attr_defs[i].type_index == phi::AttributeType::STRING) { pt_kernel_context->EmplaceBackAttr( BOOST_GET_CONST(std::string, attr_it->second)); - } else if (attr_defs[i].type_index == - std::type_index(typeid(phi::DataType))) { + } else if (attr_defs[i].type_index == phi::AttributeType::DATA_TYPE) { auto data_type = paddle::framework::TransToPhiDataType( static_cast( BOOST_GET_CONST(int, attr_it->second))); pt_kernel_context->EmplaceBackAttr(data_type); - } else if (attr_defs[i].type_index == - std::type_index(typeid(std::vector))) { - if (std::type_index(attr_it->second.type()) == - std::type_index(typeid(std::vector))) { + } else if (attr_defs[i].type_index == phi::AttributeType::INT64S) { + if (AttrTypeID(attr_it->second) == proto::AttrType::LONGS) { pt_kernel_context->EmplaceBackAttr( BOOST_GET_CONST(std::vector, attr_it->second)); - } else if (std::type_index(attr_it->second.type()) == - std::type_index(typeid(std::vector))) { + } else if (AttrTypeID(attr_it->second) == proto::AttrType::INTS) { // Emplace Back Attr according to the type of Phi_Kernel args. const auto& vector_int_attr = BOOST_GET_CONST(std::vector, attr_it->second); @@ -2514,17 +2611,14 @@ void OperatorWithKernel::BuildPhiKernelContext( vector_int_attr.end()); pt_kernel_context->EmplaceBackAttr(vector_int64_attr); } - } else if (attr_defs[i].type_index == - std::type_index(typeid(std::vector))) { + } else if (attr_defs[i].type_index == phi::AttributeType::INT32S) { const auto& vector_int_attr = BOOST_GET_CONST(std::vector, attr_it->second); pt_kernel_context->EmplaceBackAttr(vector_int_attr); - } else if (attr_defs[i].type_index == - std::type_index(typeid(std::vector))) { + } else if (attr_defs[i].type_index == phi::AttributeType::STRINGS) { pt_kernel_context->EmplaceBackAttr( BOOST_GET_CONST(std::vector, attr_it->second)); - } else if (attr_defs[i].type_index == - std::type_index(typeid(std::vector))) { + } else if (attr_defs[i].type_index == phi::AttributeType::FLOAT32S) { pt_kernel_context->EmplaceBackAttr( BOOST_GET_CONST(std::vector, attr_it->second)); } else { diff --git a/paddle/fluid/framework/operator.h b/paddle/fluid/framework/operator.h index f0887eb919c30..2e00e07535b1d 100644 --- a/paddle/fluid/framework/operator.h +++ b/paddle/fluid/framework/operator.h @@ -43,7 +43,6 @@ limitations under the License. */ #include "paddle/fluid/framework/convert_utils.h" #include "paddle/phi/core/compat/arg_map_context.h" #include "paddle/phi/core/compat/op_utils.h" -#include "paddle/phi/core/kernel_context.h" #include "paddle/phi/core/kernel_factory.h" namespace paddle { @@ -55,6 +54,10 @@ class Variable; } // namespace framework } // namespace paddle +namespace phi { +class KernelContext; +} + DECLARE_int32(inner_op_parallelism); namespace paddle { @@ -330,12 +333,12 @@ class ExecutionContext { return it->second; } - virtual std::vector InNameList() const { - std::vector vec_temp; + virtual paddle::small_vector InNameList() const { + paddle::small_vector vec_temp; vec_temp.reserve(ctx_.inputs.size()); for (auto& input : ctx_.inputs) { - vec_temp.push_back(input.first); + vec_temp.push_back(&input.first); } return vec_temp; @@ -476,6 +479,11 @@ class ExecutionArgumentMappingContext : public phi::ArgumentMappingContext { } bool IsDenseTensorInput(const std::string& name) const override { + const auto* var = ctx_.InputVar(name); + return var->IsType(); + } + + bool IsDenseTensorInputs(const std::string& name) const override { auto vars = ctx_.MultiInputVar(name); return std::all_of(vars.begin(), vars.end(), [](const Variable* var) { return var->IsType(); @@ -483,10 +491,8 @@ class ExecutionArgumentMappingContext : public phi::ArgumentMappingContext { } bool IsSelectedRowsInput(const std::string& name) const override { - auto vars = ctx_.MultiInputVar(name); - return std::all_of(vars.begin(), vars.end(), [](const Variable* var) { - return var->IsType(); - }); + const auto* var = ctx_.InputVar(name); + return var->IsType(); } bool IsDenseTensorVectorInput(const std::string& name) const override { @@ -629,7 +635,7 @@ class OperatorWithKernel : public OperatorBase { phi::KernelContext* pt_kernel_context) const; phi::KernelSignature* PhiKernelSignature() const { - return pt_kernel_signature_.get(); + return kernel_signature_.get(); } phi::Kernel* PhiKernel() const { return pt_kernel_.get(); } @@ -677,9 +683,11 @@ class OperatorWithKernel : public OperatorBase { // By default all input data must be same. proto::VarType::Type IndicateDataType(const ExecutionContext& ctx) const; // used for IndicateDataType - void ParseInputDataType(const std::vector& vars, - const std::string& name, + void ParseInputDataType(const Variable* vars, const std::string& name, proto::VarType::Type* data_type) const; + void ParseMultiInputDataType(const std::vector& vars, + const std::string& name, + proto::VarType::Type* data_type) const; // used for IndicateOrPromoteVarDataTypes Tensor* GetTensorFormInputSafely(const ExecutionContext& ctx, const std::string& name) const; @@ -699,7 +707,7 @@ class OperatorWithKernel : public OperatorBase { // we may polish the implementation here mutable bool run_phi_kernel_ = false; mutable bool run_kp_kernel = false; - mutable std::unique_ptr pt_kernel_signature_; + mutable std::unique_ptr kernel_signature_; mutable std::unique_ptr pt_kernel_; mutable std::unique_ptr arg_map_fn_; }; diff --git a/paddle/fluid/framework/phi_utils.cc b/paddle/fluid/framework/phi_utils.cc index 75bab0594758b..3eda00006f959 100644 --- a/paddle/fluid/framework/phi_utils.cc +++ b/paddle/fluid/framework/phi_utils.cc @@ -41,11 +41,11 @@ class KernelArgsNameMakerByOpProto : public KernelArgsNameMaker { ~KernelArgsNameMakerByOpProto() {} - const paddle::SmallVector& GetInputArgsNames() override; - const paddle::SmallVector& GetOutputArgsNames() override; - const paddle::SmallVector& GetAttrsArgsNames() override; + const paddle::small_vector& GetInputArgsNames() override; + const paddle::small_vector& GetOutputArgsNames() override; + const paddle::small_vector& GetAttrsArgsNames() override; - KernelSignature GetKernelSignature(); + phi::KernelSignature GetKernelSignature(); private: DISABLE_COPY_AND_ASSIGN(KernelArgsNameMakerByOpProto); @@ -53,9 +53,9 @@ class KernelArgsNameMakerByOpProto : public KernelArgsNameMaker { private: const framework::proto::OpProto* op_proto_; - paddle::SmallVector input_names_; - paddle::SmallVector output_names_; - paddle::SmallVector attr_names_; + paddle::small_vector input_names_; + paddle::small_vector output_names_; + paddle::small_vector attr_names_; }; OpKernelType TransPhiKernelKeyToOpKernelType(const phi::KernelKey& kernel_key) { @@ -81,19 +81,21 @@ OpKernelType TransPhiKernelKeyToOpKernelType(const phi::KernelKey& kernel_key) { phi::KernelKey TransOpKernelTypeToPhiKernelKey( const OpKernelType& kernel_type) { phi::Backend backend = phi::TransToPhiBackend(kernel_type.place_); - if (kernel_type.library_type_ == LibraryType::kMKLDNN) { - backend = phi::Backend::MKLDNN; - } else if (kernel_type.library_type_ == LibraryType::kCUDNN) { - backend = phi::Backend::GPUDNN; - } else if (kernel_type.library_type_ == LibraryType::kKP) { - backend = phi::Backend::KPS; - } else { - // do nothing + switch (kernel_type.library_type_) { + case LibraryType::kCUDNN: + backend = phi::Backend::GPUDNN; + break; + case LibraryType::kMKLDNN: + backend = phi::Backend::MKLDNN; + break; + case LibraryType::kKP: + backend = phi::Backend::KPS; + break; + default: + break; } - paddle::experimental::DataLayout layout = kernel_type.data_layout_; - paddle::experimental::DataType dtype = - paddle::framework::TransToPhiDataType(kernel_type.data_type_); - return phi::KernelKey(backend, layout, dtype); + return phi::KernelKey(backend, kernel_type.data_layout_, + framework::TransToPhiDataType(kernel_type.data_type_)); } phi::KernelKey FallBackToCpu(const OpKernelType& expected_kernel_key, @@ -149,7 +151,7 @@ phi::KernelKey FallBackToCpu(const OpKernelType& expected_kernel_key, return phi::KernelKey(); } -const paddle::SmallVector& +const paddle::small_vector& KernelArgsNameMakerByOpProto::GetInputArgsNames() { for (int i = 0; i < op_proto_->inputs_size(); ++i) { auto& in = op_proto_->inputs()[i]; @@ -174,7 +176,7 @@ KernelArgsNameMakerByOpProto::GetInputArgsNames() { return input_names_; } -const paddle::SmallVector& +const paddle::small_vector& KernelArgsNameMakerByOpProto::GetOutputArgsNames() { for (int i = 0; i < op_proto_->outputs_size(); ++i) { auto& out = op_proto_->outputs()[i]; @@ -194,7 +196,7 @@ KernelArgsNameMakerByOpProto::GetOutputArgsNames() { return output_names_; } -const paddle::SmallVector& +const paddle::small_vector& KernelArgsNameMakerByOpProto::GetAttrsArgsNames() { for (int i = 0; i < op_proto_->attrs_size(); ++i) { auto& attr = op_proto_->attrs()[i]; @@ -221,10 +223,10 @@ KernelArgsNameMakerByOpProto::GetAttrsArgsNames() { return attr_names_; } -KernelSignature KernelArgsNameMakerByOpProto::GetKernelSignature() { - return KernelSignature(phi::TransToPhiKernelName(op_proto_->type()).c_str(), - GetInputArgsNames(), GetAttrsArgsNames(), - GetOutputArgsNames()); +phi::KernelSignature KernelArgsNameMakerByOpProto::GetKernelSignature() { + return phi::KernelSignature( + phi::TransToPhiKernelName(op_proto_->type()).c_str(), GetInputArgsNames(), + GetAttrsArgsNames(), GetOutputArgsNames()); } std::once_flag kernel_sig_map_init_flag; diff --git a/paddle/fluid/framework/phi_utils.h b/paddle/fluid/framework/phi_utils.h index 392a3f9b06b3c..785ede5c60175 100644 --- a/paddle/fluid/framework/phi_utils.h +++ b/paddle/fluid/framework/phi_utils.h @@ -40,8 +40,6 @@ limitations under the License. */ namespace paddle { namespace framework { -using KernelSignature = phi::KernelSignature; - /* Kernel Key translate */ OpKernelType TransPhiKernelKeyToOpKernelType(const phi::KernelKey& kernel_key); @@ -55,9 +53,9 @@ phi::KernelKey FallBackToCpu(const OpKernelType& expected_kernel_key, class KernelArgsNameMaker { public: virtual ~KernelArgsNameMaker() {} - virtual const paddle::SmallVector& GetInputArgsNames() = 0; - virtual const paddle::SmallVector& GetOutputArgsNames() = 0; - virtual const paddle::SmallVector& GetAttrsArgsNames() = 0; + virtual const paddle::small_vector& GetInputArgsNames() = 0; + virtual const paddle::small_vector& GetOutputArgsNames() = 0; + virtual const paddle::small_vector& GetAttrsArgsNames() = 0; }; void InitDefaultKernelSignatureMap(); diff --git a/paddle/fluid/framework/ps_gpu_trainer.cc b/paddle/fluid/framework/ps_gpu_trainer.cc index e4004c2fbf3b5..9b12870a2bb9b 100644 --- a/paddle/fluid/framework/ps_gpu_trainer.cc +++ b/paddle/fluid/framework/ps_gpu_trainer.cc @@ -23,7 +23,8 @@ limitations under the License. */ #include "paddle/fluid/framework/device_worker_factory.h" #include "paddle/fluid/framework/fleet/ps_gpu_wrapper.h" #include "paddle/fluid/framework/trainer.h" -#if (defined PADDLE_WITH_NCCL || defined PADDLE_WITH_RCCL) && \ +#if (defined PADDLE_WITH_NCCL || defined PADDLE_WITH_RCCL || \ + defined PADDLE_WITH_XPU_BKCL) && \ (defined PADDLE_WITH_PSLIB) #ifdef PADDLE_WITH_CUDA #include "paddle/fluid/platform/cuda_device_guard.h" diff --git a/paddle/fluid/framework/ps_gpu_worker.cc b/paddle/fluid/framework/ps_gpu_worker.cc index 452c960166cb2..ad1ddbfabd091 100644 --- a/paddle/fluid/framework/ps_gpu_worker.cc +++ b/paddle/fluid/framework/ps_gpu_worker.cc @@ -18,7 +18,8 @@ limitations under the License. */ #include "paddle/fluid/platform/lodtensor_printer.h" #include "paddle/fluid/string/string_helper.h" -#if (defined PADDLE_WITH_NCCL || defined PADDLE_WITH_RCCL) && \ +#if (defined PADDLE_WITH_NCCL || defined PADDLE_WITH_RCCL || \ + defined PADDLE_WITH_XPU_BKCL) && \ (defined PADDLE_WITH_PSLIB) #ifdef PADDLE_WITH_CUDA #include "paddle/fluid/platform/cuda_device_guard.h" @@ -132,8 +133,11 @@ void PSGPUWorker::TrainFiles() { device_reader_->Start(); int cur_batch; int batch_cnt = 0; - +#if defined(PADDLE_WITH_NCCL) || defined(PADDLE_WITH_RCCL) platform::SetDeviceId(thread_id_); +#elif defined(PADDLE_WITH_XPU_BKCL) + platform::SetXPUDeviceId(thread_id_); +#endif while ((cur_batch = device_reader_->Next()) > 0) { total_ins_num += cur_batch; for (auto& op : ops_) { @@ -230,7 +234,11 @@ void PSGPUWorker::TrainFilesWithProfiler() { int total_ins_num = 0; int cur_batch; timeline.Start(); +#if defined(PADDLE_WITH_NCCL) || defined(PADDLE_WITH_RCCL) platform::SetDeviceId(thread_id_); +#elif defined(PADDLE_WITH_XPU_BKCL) + platform::SetXPUDeviceId(thread_id_); +#endif while ((cur_batch = device_reader_->Next()) > 0) { total_ins_num += cur_batch; timeline.Pause(); diff --git a/paddle/fluid/framework/shape_inference.h b/paddle/fluid/framework/shape_inference.h index bf9731bafce64..44f0ce0165c5b 100644 --- a/paddle/fluid/framework/shape_inference.h +++ b/paddle/fluid/framework/shape_inference.h @@ -65,6 +65,8 @@ class InferShapeContext { virtual bool HasOutput(const std::string &name) const = 0; virtual bool HasAttr(const std::string &name) const = 0; + virtual proto::VarType::Type GetInputVarType( + const std::string &name) const = 0; virtual std::vector GetInputsVarType( const std::string &name) const = 0; virtual std::vector GetOutputsVarType( @@ -108,11 +110,15 @@ class InferShapeContext { virtual bool IsRunMKLDNNKernel() const = 0; - virtual paddle::SmallVector + virtual paddle::small_vector GetInputVarPtrs(const std::string &name) const = 0; - virtual paddle::SmallVector + virtual paddle::small_vector GetOutputVarPtrs(const std::string &name) const = 0; + virtual const phi::ArgumentMappingFn *GetPhiArgumentMappingFn() const = 0; + + virtual const phi::KernelSignature *GetPhiDefaultKernelSignature() const = 0; + protected: virtual std::vector GetRepeatedDims(const std::string &name) const = 0; virtual void SetRepeatedDims(const std::string &name, diff --git a/paddle/fluid/framework/trainer.h b/paddle/fluid/framework/trainer.h index 8a11775702e57..2496d4d040e2e 100644 --- a/paddle/fluid/framework/trainer.h +++ b/paddle/fluid/framework/trainer.h @@ -248,7 +248,8 @@ class HeterXpuTrainer : public TrainerBase { #endif -#if (defined PADDLE_WITH_NCCL || defined PADDLE_WITH_RCCL) && \ +#if (defined PADDLE_WITH_NCCL || defined PADDLE_WITH_RCCL || \ + defined PADDLE_WITH_XPU_BKCL) && \ (defined PADDLE_WITH_PSLIB) class PSGPUTrainer : public TrainerBase { public: diff --git a/paddle/fluid/framework/trainer_factory.cc b/paddle/fluid/framework/trainer_factory.cc index f189d0213da88..1f1122d32f5c3 100644 --- a/paddle/fluid/framework/trainer_factory.cc +++ b/paddle/fluid/framework/trainer_factory.cc @@ -76,7 +76,8 @@ REGISTER_TRAINER_CLASS(HeterPipelineTrainer); (defined PADDLE_WITH_PSLIB) && (!defined(PADDLE_WITH_HETERPS)) REGISTER_TRAINER_CLASS(HeterXpuTrainer); #endif -#if (defined PADDLE_WITH_NCCL || defined PADDLE_WITH_RCCL) && \ +#if (defined PADDLE_WITH_NCCL || defined PADDLE_WITH_RCCL || \ + defined PADDLE_WITH_XPU_BKCL) && \ (defined PADDLE_WITH_PSLIB) REGISTER_TRAINER_CLASS(PSGPUTrainer); #endif diff --git a/paddle/fluid/framework/var_desc.h b/paddle/fluid/framework/var_desc.h index a20ef58f9c95f..0f8c10604f39a 100644 --- a/paddle/fluid/framework/var_desc.h +++ b/paddle/fluid/framework/var_desc.h @@ -74,6 +74,12 @@ class VarDesc { : desc_(other.desc_), attrs_(other.attrs_), original_id_(other.original_id_) {} + VarDesc &operator=(const VarDesc &other) { + desc_ = other.desc_; + attrs_ = other.attrs_; + original_id_ = other.original_id_; + return *this; + } proto::VarDesc *Proto() { return &desc_; } diff --git a/paddle/fluid/imperative/CMakeLists.txt b/paddle/fluid/imperative/CMakeLists.txt index 86044aabfd9a9..c388e32a3f4d3 100644 --- a/paddle/fluid/imperative/CMakeLists.txt +++ b/paddle/fluid/imperative/CMakeLists.txt @@ -2,9 +2,9 @@ cc_library(imperative_flag SRCS flags.cc DEPS gflags flags) cc_library(var_helper SRCS var_helper.cc DEPS tensor phi_api) cc_library(offload_scheduler SRCS offload_scheduler.cc DEPS nccl_tool layer tensor device_context) IF(WITH_XPU) -cc_library(prepared_operator SRCS prepared_operator.cc DEPS xpu_op_list proto_desc operator device_context lod_tensor selected_rows_utils var_type_traits op_kernel_type data_transform nan_inf_utils phi_api phi phi_utils var_helper) +cc_library(prepared_operator SRCS prepared_operator.cc DEPS xpu_op_list proto_desc operator device_context lod_tensor selected_rows_utils var_type_traits op_kernel_type data_transform nan_inf_utils phi_api phi_utils var_helper) ELSE() -cc_library(prepared_operator SRCS prepared_operator.cc DEPS proto_desc operator device_context lod_tensor selected_rows_utils var_type_traits op_kernel_type data_transform nan_inf_utils phi_api phi phi_utils var_helper) +cc_library(prepared_operator SRCS prepared_operator.cc DEPS proto_desc operator device_context lod_tensor selected_rows_utils var_type_traits op_kernel_type data_transform nan_inf_utils phi_api phi_utils var_helper) ENDIF() cc_library(layer SRCS layer.cc DEPS prepared_operator math_function imperative_flag variable_helper op_registry var_helper phi_api) add_subdirectory(jit) diff --git a/paddle/fluid/imperative/amp_auto_cast.cc b/paddle/fluid/imperative/amp_auto_cast.cc index 7d60b7d26f3fb..3f6863d642cc8 100644 --- a/paddle/fluid/imperative/amp_auto_cast.cc +++ b/paddle/fluid/imperative/amp_auto_cast.cc @@ -220,6 +220,7 @@ inline bool NeedCast(const std::shared_ptr& var) { paddle::platform::is_cuda_pinned_place(place) || paddle::platform::is_xpu_place(place) || paddle::platform::is_mlu_place(place) || + paddle::platform::is_custom_place(place) || paddle::platform::is_npu_place(place) || paddle::platform::is_npu_pinned_place(place)) { // CudaPinndePlace is added for varbase created by dataloader diff --git a/paddle/fluid/imperative/execution_context.h b/paddle/fluid/imperative/execution_context.h index fbc47f81fd331..124c31df73349 100644 --- a/paddle/fluid/imperative/execution_context.h +++ b/paddle/fluid/imperative/execution_context.h @@ -117,12 +117,12 @@ class DygraphExecutionContext : public framework::ExecutionContext { return it->second; } - std::vector InNameList() const override { - std::vector vec_temp; + paddle::small_vector InNameList() const override { + paddle::small_vector vec_temp; vec_temp.reserve(var_map_in_.size()); for (auto& v : var_map_in_) { - vec_temp.push_back(v.first); + vec_temp.push_back(&v.first); } return vec_temp; @@ -144,11 +144,19 @@ class DygraphExecutionContext : public framework::ExecutionContext { } size_t InputSize(const std::string& name) const override { - return InputNames(name).size(); + auto it = var_map_in_.find(name); + PADDLE_ENFORCE_NE( + it, var_map_in_.end(), + platform::errors::NotFound("Can not find [%s] in Input", name)); + return it->second.size(); } size_t OutputSize(const std::string& name) const override { - return OutputNames(name).size(); + auto it = var_map_out_.find(name); + PADDLE_ENFORCE_NE( + it, var_map_out_.end(), + platform::errors::NotFound("Can not find [%s] in Output", name)); + return it->second.size(); } const Variable* InputVar(const std::string& name) const override { diff --git a/paddle/fluid/imperative/infer_shape_context.h b/paddle/fluid/imperative/infer_shape_context.h index 5b63334c9ea99..b5df973869a9f 100644 --- a/paddle/fluid/imperative/infer_shape_context.h +++ b/paddle/fluid/imperative/infer_shape_context.h @@ -37,13 +37,17 @@ class DygraphInferShapeContext : public framework::InferShapeContext { const NameVarMap* in, const NameVarMap* out, const framework::AttributeMap* attr, const framework::AttributeMap* default_attr, const std::string op_type, - const framework::OpKernelType* op_kernel_type = nullptr) + const framework::OpKernelType* op_kernel_type = nullptr, + const phi::ArgumentMappingFn* arg_map_fn = nullptr, + const phi::KernelSignature* default_kernel_signature = nullptr) : var_map_in_(in), var_map_out_(out), attrs_(attr), default_attrs_(default_attr), op_type_(op_type), - op_kernel_type_(op_kernel_type) {} + op_kernel_type_(op_kernel_type), + arg_map_fn_(arg_map_fn), + default_kernel_signature_(default_kernel_signature) {} bool HasInput(const std::string& name) const override { // has only one input @@ -235,9 +239,10 @@ class DygraphInferShapeContext : public framework::InferShapeContext { (op_kernel_type_->data_layout_ == framework::DataLayout::kMKLDNN)); } - paddle::SmallVector + paddle::small_vector GetInputVarPtrs(const std::string& name) const override { - paddle::SmallVector + paddle::small_vector res; auto it = var_map_in_->find(name); PADDLE_ENFORCE_NE( @@ -249,10 +254,10 @@ class DygraphInferShapeContext : public framework::InferShapeContext { return res; } - paddle::SmallVector + paddle::small_vector GetOutputVarPtrs(const std::string& name) const override { - paddle::SmallVector + paddle::small_vector res; auto it = var_map_out_->find(name); PADDLE_ENFORCE_NE( @@ -296,6 +301,15 @@ class DygraphInferShapeContext : public framework::InferShapeContext { return vec_res; } + framework::proto::VarType::Type GetInputVarType( + const std::string& name) const override { + auto it = var_map_in_->find(name); + PADDLE_ENFORCE_NE( + it, var_map_in_->end(), + platform::errors::NotFound("can not find [%s] in input", name)); + return framework::ToVarType(it->second[0]->Var().Type()); + } + std::vector GetInputsVarType( const std::string& name) const override { std::vector vec_res; @@ -377,6 +391,14 @@ class DygraphInferShapeContext : public framework::InferShapeContext { "SetLoDLevel function not support in dygraph mode")); } + const phi::ArgumentMappingFn* GetPhiArgumentMappingFn() const override { + return arg_map_fn_; + } + + const phi::KernelSignature* GetPhiDefaultKernelSignature() const override { + return default_kernel_signature_; + } + protected: DDim GetDim(framework::Variable* var) const { PADDLE_ENFORCE_NOT_NULL(var, platform::errors::PreconditionNotMet( @@ -438,6 +460,9 @@ class DygraphInferShapeContext : public framework::InferShapeContext { const framework::AttributeMap* default_attrs_; const std::string op_type_; const framework::OpKernelType* op_kernel_type_; + // arg_map_fn_ and default_kernel_signature_ may be nullptr + const phi::ArgumentMappingFn* arg_map_fn_; + const phi::KernelSignature* default_kernel_signature_; }; } // namespace imperative diff --git a/paddle/fluid/imperative/layer.cc b/paddle/fluid/imperative/layer.cc index 038ea575247d5..e928cbb654839 100644 --- a/paddle/fluid/imperative/layer.cc +++ b/paddle/fluid/imperative/layer.cc @@ -459,7 +459,7 @@ static void OpBaseRunImpl(const framework::OperatorBase& op, const framework::AttributeMap& attrs, const framework::AttributeMap& default_attrs, const platform::Place& place) { - auto* op_kernel = dynamic_cast(&op); + auto* op_kernel = static_cast(&op); PADDLE_ENFORCE_NOT_NULL( op_kernel, platform::errors::PermissionDenied( "Only support operator with kernel in Dygraph mode.")); diff --git a/paddle/fluid/imperative/prepared_operator.cc b/paddle/fluid/imperative/prepared_operator.cc index cef7417ea4195..38180ba963c38 100644 --- a/paddle/fluid/imperative/prepared_operator.cc +++ b/paddle/fluid/imperative/prepared_operator.cc @@ -37,6 +37,15 @@ namespace paddle { namespace imperative { static const phi::Kernel empty_kernel; +static const framework::RuntimeContext empty_ctx({}, {}); +static const framework::Scope empty_scope; + +const phi::KernelFactory& PreparedOp::phi_kernel_factory = + phi::KernelFactory::Instance(); +const phi::OpUtilsMap& PreparedOp::phi_op_utils_map = + phi::OpUtilsMap::Instance(); +const phi::DefaultKernelSignatureMap& PreparedOp::default_phi_kernel_sig_map = + phi::DefaultKernelSignatureMap::Instance(); const std::shared_ptr& GetVariableWrapper( const std::shared_ptr& var) { @@ -105,19 +114,25 @@ PreparedOp::PreparedOp(const framework::OperatorBase& op, const framework::RuntimeContext& ctx, const framework::OpKernelType& kernel_type, const framework::OperatorWithKernel::OpKernelFunc& func, + const phi::ArgumentMappingFn* arg_map_fn, + const phi::KernelSignature* default_kernel_signature, platform::DeviceContext* dev_ctx) : op_(op), ctx_(ctx), kernel_type_(kernel_type), func_(func), dev_ctx_(dev_ctx), - pt_kernel_(empty_kernel) {} + arg_map_fn_(arg_map_fn), + default_kernel_signature_(default_kernel_signature), + phi_kernel_(empty_kernel) {} PreparedOp::PreparedOp(const framework::OperatorBase& op, const framework::RuntimeContext& ctx, const framework::OpKernelType& kernel_type, - framework::KernelSignature&& kernel_signature, - const phi::Kernel& pt_kernel, + const phi::ArgumentMappingFn* arg_map_fn, + const phi::KernelSignature* default_kernel_signature, + phi::KernelSignature&& kernel_signature, + const phi::Kernel& phi_kernel, platform::DeviceContext* dev_ctx) : op_(op), ctx_(ctx), @@ -125,21 +140,23 @@ PreparedOp::PreparedOp(const framework::OperatorBase& op, func_(nullptr), dev_ctx_(dev_ctx), run_phi_kernel_(true), - pt_kernel_signature_(std::move(kernel_signature)), - pt_kernel_(pt_kernel) {} + arg_map_fn_(arg_map_fn), + default_kernel_signature_(default_kernel_signature), + kernel_signature_(std::move(kernel_signature)), + phi_kernel_(phi_kernel) {} template -PreparedOp PrepareImpl(const NameVarMap& ins, - const NameVarMap& outs, - const framework::OperatorWithKernel& op, - const platform::Place& place, - const framework::AttributeMap& attrs, - const framework::AttributeMap& default_attrs) { +PreparedOp PrepareImpl( + const NameVarMap& ins, const NameVarMap& outs, + const framework::OperatorWithKernel& op, const platform::Place& place, + const framework::AttributeMap& attrs, + const framework::AttributeMap& default_attrs, + const phi::KernelFactory& phi_kernel_factory, + const phi::OpUtilsMap& phi_op_utils_map, + const phi::DefaultKernelSignatureMap& default_phi_kernel_sig_map) { platform::DeviceContextPool& pool = platform::DeviceContextPool::Instance(); auto* dev_ctx = pool.Get(place); - framework::RuntimeContext ctx({}, {}); - #ifdef PADDLE_WITH_MKLDNN // MKLDNN variant of code reads attributes in some of GetKernelTypeForVar and // GetKernelType functions, so we need to copy the attributes there. @@ -158,10 +175,11 @@ PreparedOp PrepareImpl(const NameVarMap& ins, // 1. get expected kernel key auto dygraph_exe_ctx = DygraphExecutionContext( - op, framework::Scope(), *dev_ctx, ctx, ins, outs, attrs, default_attrs); + op, empty_scope, *dev_ctx, empty_ctx, ins, outs, attrs, default_attrs); auto expected_kernel_key = op.GetExpectedKernelType(dygraph_exe_ctx); - framework::KernelSignature pt_kernel_signature; + const phi::KernelSignature* default_kernel_signature = nullptr; + phi::KernelSignature kernel_signature; phi::KernelKey pt_kernel_key; std::string pt_kernel_name; #if defined(PADDLE_WITH_XPU) @@ -172,12 +190,27 @@ PreparedOp PrepareImpl(const NameVarMap& ins, paddle::platform::is_in_xpu_black_list(op.Type()); #endif - if (phi::KernelFactory::Instance().HasCompatiblePhiKernel(op.Type())) { - pt_kernel_signature = - std::move(op.GetExpectedPhiKernelArgs(dygraph_exe_ctx)); - VLOG(6) << pt_kernel_signature; - pt_kernel_name = pt_kernel_signature.name; + bool has_phi_kernel = false; + + const auto* arg_map_fn = phi_op_utils_map.GetArgumentMappingFn(op.Type()); + + if (arg_map_fn) { + has_phi_kernel = true; + kernel_signature = (*arg_map_fn)( + framework::ExecutionArgumentMappingContext(dygraph_exe_ctx)); + } else { + default_kernel_signature = + default_phi_kernel_sig_map.GetNullable(op.Type()); + if (default_kernel_signature) { + has_phi_kernel = true; + kernel_signature = *default_kernel_signature; + } + } + + if (has_phi_kernel) { + VLOG(6) << kernel_signature; + pt_kernel_name = kernel_signature.name; // NOTE(Liu-xiandong): The register kernel used KP have library_type[KP], // But the default library_type is Plain, so we need to modify the // library_type here, otherwise it can't work. @@ -200,39 +233,43 @@ PreparedOp PrepareImpl(const NameVarMap& ins, auto expected_kernel_key_library_type = expected_kernel_key.library_type_; expected_kernel_key.library_type_ = paddle::framework::LibraryType::kKP; - VLOG(3) << "modifing XPU KP kernel: " << op.Type() + VLOG(3) << "modifing XPU KP kernel: " << pt_kernel_name << ", using_kernel_key:" << expected_kernel_key; + phi::KernelKey try_pt_kernel_key = TransOpKernelTypeToPhiKernelKey(expected_kernel_key); - if (!phi::KernelFactory::Instance().HasKernel(pt_kernel_name, - try_pt_kernel_key)) { + if (!phi_kernel_factory.HasKernel(pt_kernel_name, try_pt_kernel_key)) { expected_kernel_key.library_type_ = expected_kernel_key_library_type; - VLOG(3) << "modify XPU KP kernel: " << op.Type() << " is failed " - << expected_kernel_key; + VLOG(3) << "modify XPU KP kernel: " << pt_kernel_name + << " in dynamic graph is failed " << expected_kernel_key; + } else { + VLOG(3) << "modify XPU KP kernel: " << pt_kernel_name + << " in dynamic graph is succeed " << expected_kernel_key; } } } #endif pt_kernel_key = TransOpKernelTypeToPhiKernelKey(expected_kernel_key); - auto& pt_kernel = phi::KernelFactory::Instance().SelectKernel( - pt_kernel_name, pt_kernel_key); + auto& phi_kernel = + phi_kernel_factory.SelectKernel(pt_kernel_name, pt_kernel_key); - if (pt_kernel.IsValid() + if (phi_kernel.IsValid() #if defined(PADDLE_WITH_XPU) && !defined(PADDLE_WITH_XPU_KP) && !is_xpu_unsupport #endif ) { VLOG(6) << "Dynamic mode PrepareImpl - kernel name: " << pt_kernel_name << " | kernel key: " << pt_kernel_key - << " | kernel: " << pt_kernel; + << " | kernel: " << phi_kernel; if (expected_kernel_key.place_ != place) { dev_ctx = pool.Get(expected_kernel_key.place_); } - return PreparedOp(op, ctx, expected_kernel_key, - std::move(pt_kernel_signature), pt_kernel, dev_ctx); + return PreparedOp(op, empty_ctx, expected_kernel_key, arg_map_fn, + default_kernel_signature, std::move(kernel_signature), + phi_kernel, dev_ctx); } else { VLOG(6) << "Dynamic mode ChoosePhiKernel - kernel `" << pt_kernel_name << "` not found."; @@ -270,19 +307,19 @@ PreparedOp PrepareImpl(const NameVarMap& ins, || (is_xpu_unsupport && !is_xpu_kp_support) #endif ) { - if (phi::KernelFactory::Instance().HasCompatiblePhiKernel(op.Type())) { + if (has_phi_kernel) { auto pt_cpu_kernel_key = FallBackToCpu(expected_kernel_key, pt_kernel_key, op); - auto& pt_cpu_kernel = phi::KernelFactory::Instance().SelectKernel( - pt_kernel_name, pt_cpu_kernel_key); + auto& pt_cpu_kernel = + phi_kernel_factory.SelectKernel(pt_kernel_name, pt_cpu_kernel_key); if (pt_cpu_kernel.IsValid()) { VLOG(6) << "Dynamic mode PrepareImpl - kernel name: " << pt_kernel_name << " | kernel key: " << pt_cpu_kernel_key << " | kernel: " << pt_cpu_kernel; auto* cpu_ctx = pool.Get(paddle::platform::CPUPlace()); - return PreparedOp(op, ctx, expected_kernel_key, - std::move(pt_kernel_signature), pt_cpu_kernel, - cpu_ctx); + return PreparedOp(op, empty_ctx, expected_kernel_key, arg_map_fn, + default_kernel_signature, std::move(kernel_signature), + pt_cpu_kernel, cpu_ctx); } } } @@ -299,7 +336,7 @@ PreparedOp PrepareImpl(const NameVarMap& ins, #if defined(PADDLE_WITH_XPU) && !defined(PADDLE_WITH_XPU_KP) if (paddle::platform::is_xpu_place(expected_kernel_key.place_) && (kernel_iter == kernels.end() || is_xpu_unsupport)) { - VLOG(3) << "missing XPU kernel: " << op.Type() + VLOG(3) << "fluid missing XPU kernel: " << op.Type() << ", expected_kernel_key:" << expected_kernel_key << ", fallbacking to CPU one!"; expected_kernel_key.place_ = platform::CPUPlace(); @@ -310,20 +347,20 @@ PreparedOp PrepareImpl(const NameVarMap& ins, #ifdef PADDLE_WITH_XPU_KP if (paddle::platform::is_xpu_place(expected_kernel_key.place_)) { if (use_xpu_kp_kernel_rt) { - VLOG(3) << "xpu_kp using rt mode "; + VLOG(3) << "fluid xpu_kp using rt mode "; } if (use_xpu_kp_kernel_debug) { - VLOG(3) << "xpu_kp using debug mode "; + VLOG(3) << "fluid xpu_kp using debug mode "; } if (is_xpu_kp_support) { expected_kernel_key.library_type_ = paddle::framework::LibraryType::kKP; kernel_iter = kernels.find(expected_kernel_key); - VLOG(3) << "using XPU KP kernel: " << op.Type() + VLOG(3) << "using fluid XPU KP kernel: " << op.Type() << ", using_kernel_key:" << expected_kernel_key; } if (!is_xpu_kp_support && (kernel_iter == kernels.end() || is_xpu_unsupport)) { - VLOG(3) << "missing XPU kernel: " << op.Type() + VLOG(3) << "fluid missing XPU kernel: " << op.Type() << ", expected_kernel_key:" << expected_kernel_key << ", fallbacking to CPU one!"; expected_kernel_key.place_ = platform::CPUPlace(); @@ -373,7 +410,8 @@ PreparedOp PrepareImpl(const NameVarMap& ins, dev_ctx = pool.Get(expected_kernel_key.place_); } - return PreparedOp(op, ctx, expected_kernel_key, kernel_iter->second, dev_ctx); + return PreparedOp(op, empty_ctx, expected_kernel_key, kernel_iter->second, + arg_map_fn, default_kernel_signature, dev_ctx); } PreparedOp PreparedOp::Prepare(const NameVarMap& ins, @@ -382,7 +420,9 @@ PreparedOp PreparedOp::Prepare(const NameVarMap& ins, const platform::Place& place, const framework::AttributeMap& attrs, const framework::AttributeMap& default_attrs) { - return PrepareImpl(ins, outs, op, place, attrs, default_attrs); + return PrepareImpl(ins, outs, op, place, attrs, default_attrs, + phi_kernel_factory, phi_op_utils_map, + default_phi_kernel_sig_map); } PreparedOp PreparedOp::Prepare(const NameVarMap& ins, @@ -391,8 +431,9 @@ PreparedOp PreparedOp::Prepare(const NameVarMap& ins, const platform::Place& place, const framework::AttributeMap& attrs, const framework::AttributeMap& default_attrs) { - return PrepareImpl(ins, outs, op, place, attrs, - default_attrs); + return PrepareImpl( + ins, outs, op, place, attrs, default_attrs, phi_kernel_factory, + phi_op_utils_map, default_phi_kernel_sig_map); } PreparedOp PreparedOp::Prepare(const NameVarMap& ins, @@ -401,36 +442,39 @@ PreparedOp PreparedOp::Prepare(const NameVarMap& ins, const platform::Place& place, const framework::AttributeMap& attrs, const framework::AttributeMap& default_attrs) { - return PrepareImpl(ins, outs, op, place, attrs, - default_attrs); + return PrepareImpl( + ins, outs, op, place, attrs, default_attrs, phi_kernel_factory, + phi_op_utils_map, default_phi_kernel_sig_map); } template static void PreparedOpRunImpl( const framework::OperatorBase& op, const framework::RuntimeContext& ctx, const framework::OpKernelType& kernel_type, const framework::OperatorWithKernel::OpKernelFunc& func, + const phi::ArgumentMappingFn* arg_map_fn, + const phi::KernelSignature* default_kernel_signature, platform::DeviceContext* dev_ctx, const NameVarMap& ins, const NameVarMap& outs, const framework::AttributeMap& attrs, const framework::AttributeMap& default_attrs) { // TODO(zjl): remove scope in dygraph - framework::Scope scope; { - platform::RecordEvent record_event(op.Type() + "::infer_shape", + platform::RecordEvent record_event("infer_shape", platform::TracerEventType::OperatorInner, 1, platform::EventRole::kInnerOp); DygraphInferShapeContext infer_shape_ctx( - &ins, &outs, &attrs, &default_attrs, op.Type(), &kernel_type); + &ins, &outs, &attrs, &default_attrs, op.Type(), &kernel_type, + arg_map_fn, default_kernel_signature); op.Info().infer_shape_(&infer_shape_ctx); } { - platform::RecordEvent record_event(op.Type() + "::compute", + platform::RecordEvent record_event("compute", platform::TracerEventType::OperatorInner, 1, platform::EventRole::kInnerOp); - func(DygraphExecutionContext(op, scope, *dev_ctx, ctx, ins, outs, - attrs, default_attrs)); + func(DygraphExecutionContext(op, empty_scope, *dev_ctx, ctx, ins, + outs, attrs, default_attrs)); } if (FLAGS_check_nan_inf) { @@ -467,33 +511,35 @@ template static void PreparedOpRunPtImpl( const framework::OperatorBase& op, const framework::OpKernelType& kernel_type, - const framework::KernelSignature& pt_kernel_signature, - const phi::Kernel& pt_kernel, platform::DeviceContext* dev_ctx, - const NameVarMap& ins, const NameVarMap& outs, - const framework::AttributeMap& attrs, + const phi::ArgumentMappingFn* arg_map_fn, + const phi::KernelSignature* default_kernel_signature, + const phi::KernelSignature& kernel_signature, const phi::Kernel& phi_kernel, + platform::DeviceContext* dev_ctx, const NameVarMap& ins, + const NameVarMap& outs, const framework::AttributeMap& attrs, const framework::AttributeMap& default_attrs) { { - platform::RecordEvent record_event(op.Type() + "::infer_shape", + platform::RecordEvent record_event("infer_shape", platform::TracerEventType::OperatorInner, 1, platform::EventRole::kInnerOp); DygraphInferShapeContext infer_shape_ctx( - &ins, &outs, &attrs, &default_attrs, op.Type(), &kernel_type); + &ins, &outs, &attrs, &default_attrs, op.Type(), &kernel_type, + arg_map_fn, default_kernel_signature); op.Info().infer_shape_(&infer_shape_ctx); } { - platform::RecordEvent record_event(op.Type() + "::compute", + platform::RecordEvent record_event("compute", platform::TracerEventType::OperatorInner, 1, platform::EventRole::kInnerOp); - PreparePhiData(pt_kernel, pt_kernel_signature, ins); + PreparePhiData(phi_kernel, kernel_signature, ins); phi::KernelContext pt_kernel_context; - BuildDygraphPhiKernelContext(pt_kernel_signature, pt_kernel, ins, + BuildDygraphPhiKernelContext(kernel_signature, phi_kernel, ins, outs, attrs, default_attrs, dev_ctx, &pt_kernel_context); - pt_kernel(&pt_kernel_context); + phi_kernel(&pt_kernel_context); } if (FLAGS_check_nan_inf) { @@ -519,12 +565,14 @@ void PreparedOp::Run(const NameVarMap& ins, const framework::AttributeMap& attrs, const framework::AttributeMap& default_attrs) { if (run_phi_kernel_) { - PreparedOpRunPtImpl(op_, kernel_type_, pt_kernel_signature_, - pt_kernel_, dev_ctx_, ins, outs, attrs, + PreparedOpRunPtImpl(op_, kernel_type_, arg_map_fn_, + default_kernel_signature_, kernel_signature_, + phi_kernel_, dev_ctx_, ins, outs, attrs, default_attrs); } else { - PreparedOpRunImpl(op_, ctx_, kernel_type_, func_, dev_ctx_, ins, - outs, attrs, default_attrs); + PreparedOpRunImpl(op_, ctx_, kernel_type_, func_, arg_map_fn_, + default_kernel_signature_, dev_ctx_, ins, outs, + attrs, default_attrs); } } @@ -534,11 +582,13 @@ void PreparedOp::Run(const NameVarMap& ins, const framework::AttributeMap& default_attrs) { if (run_phi_kernel_) { PreparedOpRunPtImpl( - op_, kernel_type_, pt_kernel_signature_, pt_kernel_, dev_ctx_, ins, - outs, attrs, default_attrs); + op_, kernel_type_, arg_map_fn_, default_kernel_signature_, + kernel_signature_, phi_kernel_, dev_ctx_, ins, outs, attrs, + default_attrs); } else { - PreparedOpRunImpl(op_, ctx_, kernel_type_, func_, dev_ctx_, - ins, outs, attrs, default_attrs); + PreparedOpRunImpl( + op_, ctx_, kernel_type_, func_, arg_map_fn_, default_kernel_signature_, + dev_ctx_, ins, outs, attrs, default_attrs); } } @@ -548,12 +598,13 @@ void PreparedOp::Run(const NameVarMap& ins, const framework::AttributeMap& default_attrs) { if (run_phi_kernel_) { PreparedOpRunPtImpl( - op_, kernel_type_, pt_kernel_signature_, pt_kernel_, dev_ctx_, ins, - outs, attrs, default_attrs); + op_, kernel_type_, arg_map_fn_, default_kernel_signature_, + kernel_signature_, phi_kernel_, dev_ctx_, ins, outs, attrs, + default_attrs); } else { - PreparedOpRunImpl(op_, ctx_, kernel_type_, func_, - dev_ctx_, ins, outs, attrs, - default_attrs); + PreparedOpRunImpl( + op_, ctx_, kernel_type_, func_, arg_map_fn_, default_kernel_signature_, + dev_ctx_, ins, outs, attrs, default_attrs); } } diff --git a/paddle/fluid/imperative/prepared_operator.h b/paddle/fluid/imperative/prepared_operator.h index cb3275674ed49..9e729fee69d86 100644 --- a/paddle/fluid/imperative/prepared_operator.h +++ b/paddle/fluid/imperative/prepared_operator.h @@ -31,6 +31,7 @@ #include "paddle/fluid/framework/convert_utils.h" #include "paddle/phi/core/dense_tensor.h" +#include "paddle/phi/core/kernel_context.h" #include "paddle/phi/core/selected_rows.h" DECLARE_bool(use_mkldnn); @@ -149,13 +150,17 @@ class PreparedOp { const framework::RuntimeContext& ctx, const framework::OpKernelType& kernel_type, const framework::OperatorWithKernel::OpKernelFunc& func, + const phi::ArgumentMappingFn* arg_map_fn, + const phi::KernelSignature* default_kernel_signature, platform::DeviceContext* dev_ctx); PreparedOp(const framework::OperatorBase& op, const framework::RuntimeContext& ctx, const framework::OpKernelType& kernel_type, - framework::KernelSignature&& kernel_signature, - const phi::Kernel& pt_kernel, platform::DeviceContext* dev_ctx); + const phi::ArgumentMappingFn* arg_map_fn, + const phi::KernelSignature* default_kernel_signature, + phi::KernelSignature&& kernel_signature, + const phi::Kernel& phi_kernel, platform::DeviceContext* dev_ctx); static PreparedOp Prepare(const NameVarMap& ins, const NameVarMap& outs, @@ -205,8 +210,14 @@ class PreparedOp { // we may polish the implementation here bool run_phi_kernel_{false}; bool run_kp_kernel_{false}; - framework::KernelSignature pt_kernel_signature_; - const phi::Kernel& pt_kernel_; + const phi::ArgumentMappingFn* arg_map_fn_; + const phi::KernelSignature* default_kernel_signature_; + phi::KernelSignature kernel_signature_; + const phi::Kernel& phi_kernel_; + + static const phi::KernelFactory& phi_kernel_factory; + static const phi::OpUtilsMap& phi_op_utils_map; + static const phi::DefaultKernelSignatureMap& default_phi_kernel_sig_map; }; const inline framework::Attribute& GetAttr( @@ -225,21 +236,23 @@ const inline framework::Attribute& GetAttr( } template -void BuildDygraphPhiKernelContext( - const framework::KernelSignature& pt_kernel_signature, - const phi::Kernel& pt_kernel, const NameVarMap& ins, - const NameVarMap& outs, const framework::AttributeMap& attrs, - const framework::AttributeMap& default_attrs, - platform::DeviceContext* dev_ctx, phi::KernelContext* kernel_ctx) { +void BuildDygraphPhiKernelContext(const phi::KernelSignature& kernel_signature, + const phi::Kernel& phi_kernel, + const NameVarMap& ins, + const NameVarMap& outs, + const framework::AttributeMap& attrs, + const framework::AttributeMap& default_attrs, + platform::DeviceContext* dev_ctx, + phi::KernelContext* kernel_ctx) { kernel_ctx->SetDeviceContext(dev_ctx); - auto& input_names = std::get<0>(pt_kernel_signature.args); - auto& attr_names = std::get<1>(pt_kernel_signature.args); - auto& output_names = std::get<2>(pt_kernel_signature.args); + const auto& input_names = kernel_signature.input_names; + const auto& attr_names = kernel_signature.attr_names; + const auto& output_names = kernel_signature.output_names; - auto& input_defs = pt_kernel.args_def().input_defs(); - auto& output_defs = pt_kernel.args_def().output_defs(); - auto& attr_defs = pt_kernel.args_def().attribute_defs(); + auto& input_defs = phi_kernel.args_def().input_defs(); + auto& output_defs = phi_kernel.args_def().output_defs(); + auto& attr_defs = phi_kernel.args_def().attribute_defs(); PADDLE_ENFORCE_EQ(input_names.size(), input_defs.size(), platform::errors::InvalidArgument( @@ -285,7 +298,7 @@ void BuildDygraphPhiKernelContext( "Can not find input variable '%s' for %s OP, please check whether " "the name setting in OpArgumentMapping is consistent with that in " "OpMaker.", - input_names[i], pt_kernel_signature.name)); + input_names[i], kernel_signature.name)); } } @@ -302,7 +315,7 @@ void BuildDygraphPhiKernelContext( tensor_in = &(var.template Get()); kernel_ctx->EmplaceBackInputWithoutSetRange(tensor_in); } else if (var.template IsType()) { - paddle::SmallVector tensor_vector; + paddle::small_vector tensor_vector; auto& tensor_array = var.template Get(); for (auto& t : tensor_array) { tensor_vector.emplace_back(&t); @@ -348,7 +361,7 @@ void BuildDygraphPhiKernelContext( tensor_out = var->template GetMutable(); kernel_ctx->EmplaceBackOutputWithoutSetRange(tensor_out); } else if (var->template IsType()) { - paddle::SmallVector tensor_vector; + paddle::small_vector tensor_vector; auto* tensor_array = var->template GetMutable(); for (auto& t : *tensor_array) { @@ -369,28 +382,23 @@ void BuildDygraphPhiKernelContext( } for (size_t i = 0; i < attr_names.size(); ++i) { - if (attr_defs[i].type_index == std::type_index(typeid(phi::IntArray))) { + if (attr_defs[i].type_index == phi::AttributeType::INT_ARRAY) { if (attrs.find(attr_names[i]) != attrs.end()) { // shape is in the attribute auto& attr = GetAttr(attrs, default_attrs, attr_names[i]); - if (std::type_index(attr.type()) == - std::type_index(typeid(std::vector))) { + if (AttrTypeID(attr) == framework::proto::AttrType::LONGS) { kernel_ctx->EmplaceBackAttr(std::move( phi::IntArray(BOOST_GET_CONST(std::vector, attr)))); - } else if (std::type_index(attr.type()) == - std::type_index(typeid(std::vector))) { + } else if (AttrTypeID(attr) == framework::proto::AttrType::INTS) { kernel_ctx->EmplaceBackAttr(std::move( phi::IntArray(BOOST_GET_CONST(std::vector, attr)))); - } else if (std::type_index(attr.type()) == - std::type_index(typeid(int64_t))) { + } else if (AttrTypeID(attr) == framework::proto::AttrType::LONG) { kernel_ctx->EmplaceBackAttr( std::move(phi::IntArray(&BOOST_GET_CONST(int64_t, attr), 1))); - } else if (std::type_index(attr.type()) == - std::type_index(typeid(int32_t))) { + } else if (AttrTypeID(attr) == framework::proto::AttrType::INT) { kernel_ctx->EmplaceBackAttr( std::move(phi::IntArray(&BOOST_GET_CONST(int32_t, attr), 1))); - } else if (attr_defs[i].type_index == - std::type_index(typeid(std::vector))) { + } else if (attr_defs[i].type_index == phi::AttributeType::INT32S) { const auto& vector_int_attr = BOOST_GET_CONST(std::vector, attr); kernel_ctx->EmplaceBackAttr(vector_int_attr); } else { @@ -414,24 +422,20 @@ void BuildDygraphPhiKernelContext( std::move(experimental::MakePhiIntArrayFromVarList(variables))); } } - } else if (attr_defs[i].type_index == - std::type_index(typeid(phi::Scalar))) { - // TODO(chenweihang): support other attrs later + } else if (attr_defs[i].type_index == phi::AttributeType::SCALAR) { // TODO(zhangyunfei): Scalar should hold scaler type, and we should check // attribtue type by attr_defs if (attrs.find(attr_names[i]) != attrs.end() || default_attrs.find(attr_names[i]) != default_attrs.end()) { // scalar is in the attribute auto& attr = GetAttr(attrs, default_attrs, attr_names[i]); - if (std::type_index(attr.type()) == std::type_index(typeid(float))) { + if (AttrTypeID(attr) == framework::proto::AttrType::FLOAT) { kernel_ctx->EmplaceBackAttr( std::move(phi::Scalar(BOOST_GET_CONST(float, attr)))); - } else if (std::type_index(attr.type()) == - std::type_index(typeid(std::string))) { + } else if (AttrTypeID(attr) == framework::proto::AttrType::STRING) { kernel_ctx->EmplaceBackAttr( std::move(phi::Scalar(BOOST_GET_CONST(std::string, attr)))); - } else if (std::type_index(attr.type()) == - std::type_index(typeid(int))) { + } else if (AttrTypeID(attr) == framework::proto::AttrType::INT) { kernel_ctx->EmplaceBackAttr( std::move(phi::Scalar(BOOST_GET_CONST(int, attr)))); } else { @@ -451,17 +455,15 @@ void BuildDygraphPhiKernelContext( auto& ins_vector = ins.at(attr_names[i]); auto tensor_attr = experimental::MakePhiScalarFromVar(ins_vector[0]->Var()); - if (attr_defs[i].type_index == std::type_index(typeid(int))) { + if (attr_defs[i].type_index == phi::AttributeType::INT32) { int val = tensor_attr.template to(); kernel_ctx->EmplaceBackAttr(val); } else { PADDLE_THROW(platform::errors::Unimplemented("only support int here")); } - } else if (attr_defs[i].type_index == - std::type_index(typeid(std::vector))) { + } else if (attr_defs[i].type_index == phi::AttributeType::SCALARS) { auto& attr = GetAttr(attrs, default_attrs, attr_names[i]); - if (std::type_index(attr.type()) == - std::type_index(typeid(std::vector))) { + if (AttrTypeID(attr) == framework::proto::AttrType::INTS) { const auto& vec = BOOST_GET_CONST(std::vector, attr); std::vector scalar_list; scalar_list.reserve(vec.size()); @@ -469,8 +471,7 @@ void BuildDygraphPhiKernelContext( scalar_list.emplace_back(val); } kernel_ctx->EmplaceBackAttr(std::move(scalar_list)); - } else if (std::type_index(attr.type()) == - std::type_index(typeid(std::vector))) { + } else if (AttrTypeID(attr) == framework::proto::AttrType::LONGS) { const auto& vec = BOOST_GET_CONST(std::vector, attr); std::vector scalar_list; scalar_list.reserve(vec.size()); @@ -478,8 +479,7 @@ void BuildDygraphPhiKernelContext( scalar_list.emplace_back(val); } kernel_ctx->EmplaceBackAttr(std::move(scalar_list)); - } else if (std::type_index(attr.type()) == - std::type_index(typeid(std::vector))) { + } else if (AttrTypeID(attr) == framework::proto::AttrType::FLOATS) { const auto& vec = BOOST_GET_CONST(std::vector, attr); std::vector scalar_list; scalar_list.reserve(vec.size()); @@ -487,8 +487,7 @@ void BuildDygraphPhiKernelContext( scalar_list.emplace_back(val); } kernel_ctx->EmplaceBackAttr(std::move(scalar_list)); - } else if (std::type_index(attr.type()) == - std::type_index(typeid(std::vector))) { + } else if (AttrTypeID(attr) == framework::proto::AttrType::FLOAT64S) { const auto& vec = BOOST_GET_CONST(std::vector, attr); std::vector scalar_list; scalar_list.reserve(vec.size()); @@ -496,8 +495,7 @@ void BuildDygraphPhiKernelContext( scalar_list.emplace_back(val); } kernel_ctx->EmplaceBackAttr(std::move(scalar_list)); - } else if (std::type_index(attr.type()) == - std::type_index(typeid(std::vector))) { + } else if (AttrTypeID(attr) == framework::proto::AttrType::BOOLEANS) { const auto& vec = BOOST_GET_CONST(std::vector, attr); std::vector scalar_list; scalar_list.reserve(vec.size()); @@ -512,49 +510,39 @@ void BuildDygraphPhiKernelContext( attr_names[i])); } } else { - // TODO(chenweihang): support other attrs later - auto& attr = GetAttr(attrs, default_attrs, attr_names[i]); - if (attr_defs[i].type_index == std::type_index(typeid(int))) { + if (attr_defs[i].type_index == phi::AttributeType::INT32) { kernel_ctx->EmplaceBackAttr(BOOST_GET_CONST(int, attr)); - } else if (attr_defs[i].type_index == std::type_index(typeid(float))) { + } else if (attr_defs[i].type_index == phi::AttributeType::FLOAT32) { kernel_ctx->EmplaceBackAttr(BOOST_GET_CONST(float, attr)); - } else if (attr_defs[i].type_index == std::type_index(typeid(bool))) { + } else if (attr_defs[i].type_index == phi::AttributeType::BOOL) { kernel_ctx->EmplaceBackAttr(BOOST_GET_CONST(bool, attr)); - } else if (attr_defs[i].type_index == std::type_index(typeid(int64_t))) { + } else if (attr_defs[i].type_index == phi::AttributeType::INT64) { kernel_ctx->EmplaceBackAttr(BOOST_GET_CONST(int64_t, attr)); - } else if (attr_defs[i].type_index == - std::type_index(typeid(std::string))) { + } else if (attr_defs[i].type_index == phi::AttributeType::STRING) { kernel_ctx->EmplaceBackAttr(BOOST_GET_CONST(std::string, attr)); - } else if (attr_defs[i].type_index == - std::type_index(typeid(phi::DataType))) { + } else if (attr_defs[i].type_index == phi::AttributeType::DATA_TYPE) { auto data_type = framework::TransToPhiDataType( static_cast( BOOST_GET_CONST(int, attr))); kernel_ctx->EmplaceBackAttr(data_type); - } else if (attr_defs[i].type_index == - std::type_index(typeid(std::vector))) { - if (std::type_index(attr.type()) == - std::type_index(typeid(std::vector))) { + } else if (attr_defs[i].type_index == phi::AttributeType::INT64S) { + if (AttrTypeID(attr) == framework::proto::AttrType::LONGS) { kernel_ctx->EmplaceBackAttr( BOOST_GET_CONST(std::vector, attr)); - } else if (std::type_index(attr.type()) == - std::type_index(typeid(std::vector))) { + } else if (AttrTypeID(attr) == framework::proto::AttrType::INTS) { // Emplace Back Attr according to the type of Phi_Kernel args. const auto& vector_int_attr = BOOST_GET_CONST(std::vector, attr); const std::vector vector_int64_attr(vector_int_attr.begin(), vector_int_attr.end()); kernel_ctx->EmplaceBackAttr(vector_int64_attr); } - } else if (attr_defs[i].type_index == - std::type_index(typeid(std::vector))) { + } else if (attr_defs[i].type_index == phi::AttributeType::INT32S) { kernel_ctx->EmplaceBackAttr(BOOST_GET_CONST(std::vector, attr)); - } else if (attr_defs[i].type_index == - std::type_index(typeid(std::vector))) { + } else if (attr_defs[i].type_index == phi::AttributeType::STRINGS) { kernel_ctx->EmplaceBackAttr( BOOST_GET_CONST(std::vector, attr)); - } else if (attr_defs[i].type_index == - std::type_index(typeid(std::vector))) { + } else if (attr_defs[i].type_index == phi::AttributeType::FLOAT32S) { kernel_ctx->EmplaceBackAttr(BOOST_GET_CONST(std::vector, attr)); } else { PADDLE_THROW(platform::errors::Unimplemented( @@ -567,11 +555,11 @@ void BuildDygraphPhiKernelContext( } template -void PreparePhiData(const phi::Kernel& pt_kernel, - const framework::KernelSignature& pt_kernel_signature, +void PreparePhiData(const phi::Kernel& phi_kernel, + const phi::KernelSignature& kernel_signature, const NameVarMap& ins) { - auto& input_names = std::get<0>(pt_kernel_signature.args); - auto& input_defs = pt_kernel.args_def().input_defs(); + const auto& input_names = kernel_signature.input_names; + auto& input_defs = phi_kernel.args_def().input_defs(); PADDLE_ENFORCE_EQ(input_names.size(), input_defs.size(), platform::errors::InvalidArgument( diff --git a/paddle/fluid/imperative/tracer.cc b/paddle/fluid/imperative/tracer.cc index 3e2e082fbaa27..6c31b025507f8 100644 --- a/paddle/fluid/imperative/tracer.cc +++ b/paddle/fluid/imperative/tracer.cc @@ -192,7 +192,7 @@ void Tracer::TraceOpImpl(const std::string& type, paddle::framework::AttributeMap* passed_default_attrs_, bool use_default_attr_map) { platform::RecordEvent op_type_record_event( - type + " trace_op", platform::TracerEventType::Operator, 1); + "trace_op", platform::TracerEventType::Operator, 1); platform::ScopedFlushDenormal flush; VLOG(1) << "Trace Op: " << type; if (FLAGS_use_mkldnn) { @@ -320,7 +320,7 @@ void Tracer::TraceOpImpl(const std::string& type, { platform::RecordEvent node_creation_record_event( - type + " node_creation", platform::TracerEventType::OperatorInner, 1); + "grad_node_creation", platform::TracerEventType::OperatorInner, 1); if (ComputeRequiredGrad(new_ins, outs, trace_backward)) { PADDLE_ENFORCE_EQ( diff --git a/paddle/fluid/inference/CMakeLists.txt b/paddle/fluid/inference/CMakeLists.txt index bdf364aa9adcd..7fae481f58289 100644 --- a/paddle/fluid/inference/CMakeLists.txt +++ b/paddle/fluid/inference/CMakeLists.txt @@ -36,7 +36,7 @@ endif() # fluid_modules exclude API-interface of inference/api and inference/capi_exp get_property(fluid_modules GLOBAL PROPERTY FLUID_MODULES) get_property(phi_modules GLOBAL PROPERTY PHI_MODULES) -set(utils_modules stringpiece pretty_log string_helper) +set(utils_modules stringpiece pretty_log string_helper benchmark) add_subdirectory(api) @@ -46,13 +46,9 @@ set(STATIC_INFERENCE_API paddle_inference_api analysis_predictor zero_copy_tensor reset_tensor_array analysis_config paddle_pass_builder activation_functions ${mkldnn_quantizer_cfg}) -if(WITH_ONNXRUNTIME) - set(STATIC_INFERENCE_API ${STATIC_INFERENCE_API} onnxruntime_predictor) -endif() - -#TODO(wilber, T8T9): Do we still need to support windows gpu static library? +#windows GPU static library over the limit, so not create_static_lib, and cc_library is dummy if(WIN32 AND WITH_GPU) - cc_library(paddle_inference DEPS ${fluid_modules} ${phi_modules} ${STATIC_INFERENCE_API} ${utils_modules}) + cc_library(paddle_inference DEPS ${fluid_modules} phi ${STATIC_INFERENCE_API} ${utils_modules}) else() create_static_lib(paddle_inference ${fluid_modules} ${phi_modules} ${STATIC_INFERENCE_API} ${utils_modules}) endif() @@ -84,7 +80,7 @@ set(SHARED_INFERENCE_SRCS ${PADDLE_CUSTOM_OP_SRCS}) # shared inference library deps -set(SHARED_INFERENCE_DEPS ${fluid_modules} ${phi_modules} analysis_predictor) +set(SHARED_INFERENCE_DEPS ${fluid_modules} phi analysis_predictor ${utils_modules}) if (WITH_CRYPTO) set(SHARED_INFERENCE_DEPS ${SHARED_INFERENCE_DEPS} paddle_crypto) @@ -98,7 +94,6 @@ if (WITH_ONNXRUNTIME) set(SHARED_INFERENCE_SRCS ${SHARED_INFERENCE_SRCS} ${CMAKE_CURRENT_SOURCE_DIR}/api/onnxruntime_predictor.cc ) - set(SHARED_INFERENCE_DEPS ${SHARED_INFERENCE_DEPS} onnxruntime_predictor) endif (WITH_ONNXRUNTIME) # Create shared inference library diff --git a/paddle/fluid/inference/api/CMakeLists.txt b/paddle/fluid/inference/api/CMakeLists.txt index bdc16ef4c7907..edec1b1c7d0e4 100755 --- a/paddle/fluid/inference/api/CMakeLists.txt +++ b/paddle/fluid/inference/api/CMakeLists.txt @@ -50,9 +50,8 @@ if(WITH_GPU AND TENSORRT_FOUND) endif() if (WITH_ONNXRUNTIME) - cc_library(analysis_predictor SRCS analysis_predictor.cc ${mkldnn_quantizer_src} DEPS ${inference_deps} + cc_library(analysis_predictor SRCS analysis_predictor.cc onnxruntime_predictor.cc ${mkldnn_quantizer_src} DEPS ${inference_deps} zero_copy_tensor ir_pass_manager op_compatible_info infer_io_utils onnxruntime paddle2onnx) - cc_library(onnxruntime_predictor SRCS onnxruntime_predictor.cc DEPS analysis_predictor) else (WITH_ONNXRUNTIME) cc_library(analysis_predictor SRCS analysis_predictor.cc ${mkldnn_quantizer_src} DEPS ${inference_deps} zero_copy_tensor ir_pass_manager op_compatible_info infer_io_utils) @@ -82,16 +81,6 @@ elseif (WIN32) ARGS --dirname=${WORD2VEC_MODEL_DIR}) endif() -if (WITH_ONNXRUNTIME) - if (NOT APPLE AND NOT WIN32) - cc_test(test_onnxruntime_predictor SRCS onnxruntime_predictor_tester.cc DEPS paddle_inference_shared - ARGS --dirname=${MOBILENETV2_MODEL_DIR}) - elseif (WIN32) - cc_test(test_onnxruntime_predictor SRCS onnxruntime_predictor_tester.cc DEPS onnxruntime_predictor benchmark ${inference_deps} - ARGS --dirname=${MOBILENETV2_MODEL_DIR}) - endif() -endif() - if(WITH_TESTING AND WITH_MKLDNN) if (NOT APPLE AND NOT WIN32) cc_test(test_mkldnn_quantizer SRCS mkldnn_quantizer_tester.cc DEPS paddle_inference_shared ARGS --dirname=${WORD2VEC_MODEL_DIR}) diff --git a/paddle/fluid/inference/api/analysis_predictor.cc b/paddle/fluid/inference/api/analysis_predictor.cc index 015f4471a0246..4f0d4a908380f 100644 --- a/paddle/fluid/inference/api/analysis_predictor.cc +++ b/paddle/fluid/inference/api/analysis_predictor.cc @@ -48,6 +48,7 @@ #include "paddle/fluid/platform/place.h" #include "paddle/fluid/platform/profiler.h" #include "paddle/phi/api/ext/op_meta_info.h" +#include "paddle/phi/common/place.h" #include "paddle/utils/string/split.h" #if defined(PADDLE_WITH_DISTRIBUTE) && defined(PADDLE_WITH_PSCORE) @@ -1641,7 +1642,9 @@ AnalysisPredictor::~AnalysisPredictor() { StatisticShapeRangeInfo(); } - memory::Release(place_); + if (place_.GetType() != phi::AllocationType::UNDEFINED) { + memory::Release(place_); + } } std::unique_ptr AnalysisPredictor::Clone() { diff --git a/paddle/fluid/inference/api/details/CMakeLists.txt b/paddle/fluid/inference/api/details/CMakeLists.txt index b2cfb060dd325..0d7a8d57a9c5a 100644 --- a/paddle/fluid/inference/api/details/CMakeLists.txt +++ b/paddle/fluid/inference/api/details/CMakeLists.txt @@ -16,9 +16,10 @@ cc_library(reset_tensor_array SRCS reset_tensor_array.cc DEPS lod_tensor scope) if (WITH_ONNXRUNTIME) cc_library(zero_copy_tensor SRCS zero_copy_tensor.cc DEPS scope lod_tensor enforce onnxruntime) + cc_library(zero_copy_tensor_dummy SRCS zero_copy_tensor_dummy.cc DEPS onnxruntime) else (WITH_ONNXRUNTIME) cc_library(zero_copy_tensor SRCS zero_copy_tensor.cc DEPS scope lod_tensor enforce) + cc_library(zero_copy_tensor_dummy SRCS zero_copy_tensor_dummy.cc) endif (WITH_ONNXRUNTIME) -cc_library(zero_copy_tensor_dummy SRCS zero_copy_tensor_dummy.cc) cc_test(zero_copy_tensor_test SRCS zero_copy_tensor_test.cc DEPS paddle_inference_api) diff --git a/paddle/fluid/inference/tensorrt/CMakeLists.txt b/paddle/fluid/inference/tensorrt/CMakeLists.txt index d1d146b2ce5f6..c713e3a66ac71 100644 --- a/paddle/fluid/inference/tensorrt/CMakeLists.txt +++ b/paddle/fluid/inference/tensorrt/CMakeLists.txt @@ -1,8 +1,8 @@ # Compiling with WITH_PYTHON=ON and WITH_TENSORRT=ON failed on windows. Temporarily add paddle_inference_api dependency to solve the problem if(WIN32) -nv_library(tensorrt_engine SRCS engine.cc trt_int8_calibrator.cc DEPS ${GLOB_OPERATOR_DEPS} framework_proto device_context boost paddle_inference_api) + nv_library(tensorrt_engine SRCS engine.cc trt_int8_calibrator.cc DEPS ${GLOB_OPERATOR_DEPS} framework_proto device_context boost paddle_inference_api) else() -nv_library(tensorrt_engine SRCS engine.cc trt_int8_calibrator.cc DEPS ${GLOB_OPERATOR_DEPS} framework_proto device_context boost) + nv_library(tensorrt_engine SRCS engine.cc trt_int8_calibrator.cc DEPS ${GLOB_OPERATOR_DEPS} framework_proto device_context boost) endif() nv_library(tensorrt_op_teller SRCS op_teller.cc DEPS framework_proto device_context boost) nv_test(test_tensorrt SRCS test_tensorrt.cc DEPS dynload_cuda device_context dynamic_loader) diff --git a/paddle/fluid/inference/tensorrt/convert/flatten_contiguous_range_op.cc b/paddle/fluid/inference/tensorrt/convert/flatten_contiguous_range_op.cc index 706814340a0e9..e08f50833ed99 100644 --- a/paddle/fluid/inference/tensorrt/convert/flatten_contiguous_range_op.cc +++ b/paddle/fluid/inference/tensorrt/convert/flatten_contiguous_range_op.cc @@ -30,14 +30,17 @@ class FlattenContiguousRangeOpConverter : public OpConverter { public: void operator()(const framework::proto::OpDesc& op, const framework::Scope& scope, bool test_mode) override { + VLOG(3) << "convert a fluid flatten_contiguous_range op to tensorrt layer"; framework::OpDesc op_desc(op, nullptr); // Declare inputs auto* input = engine_->GetITensor(op_desc.Input("X")[0]); - int dims = input->getDimensions().nbDims; + const auto input_dim = input->getDimensions(); + const int dims = input_dim.nbDims; int start_axis = BOOST_GET_CONST(int, op_desc.GetAttr("start_axis")); int stop_axis = BOOST_GET_CONST(int, op_desc.GetAttr("stop_axis")); - nvinfer1::IShuffleLayer* layer = nullptr; + nvinfer1::IShuffleLayer* layer = + TRT_ENGINE_ADD_LAYER(engine_, Shuffle, *input); if (!engine_->with_dynamic_shape()) { if (start_axis < 0) start_axis += dims + 1; if (stop_axis < 0) stop_axis += dims + 1; @@ -46,7 +49,7 @@ class FlattenContiguousRangeOpConverter : public OpConverter { flatten_dim.nbDims = dims - (stop_axis - start_axis); for (int i = 0, j = 0; i < dims; ++i) { if (start_axis <= i + 1 && i + 1 <= stop_axis) { - int dim_i = input->getDimensions().d[i]; + int dim_i = input_dim.d[i]; PADDLE_ENFORCE_GT(dim_i, 0, platform::errors::InvalidArgument( "flatten_contiguous_range input dim " "should be > 0, but got %d.", @@ -56,72 +59,103 @@ class FlattenContiguousRangeOpConverter : public OpConverter { flatten_dim.d[j++] = dim_prod; } } else { - flatten_dim.d[j++] = input->getDimensions().d[i]; + flatten_dim.d[j++] = input_dim.d[i]; } } - layer = TRT_ENGINE_ADD_LAYER(engine_, Shuffle, *input); layer->setReshapeDimensions(flatten_dim); } else { if (start_axis < 0) start_axis += dims; if (stop_axis < 0) stop_axis += dims; - auto* shape_layer = TRT_ENGINE_ADD_LAYER(engine_, Shape, *input); - auto* shape_layer_itensor = shape_layer->getOutput(0); - nvinfer1::Dims start_dim, size_dim, stride_dim; - start_dim.nbDims = 1; - size_dim.nbDims = 1; - stride_dim.nbDims = 1; - start_dim.d[0] = start_axis; - size_dim.d[0] = stop_axis - start_axis + 1; - stride_dim.d[0] = 1; - auto* slice_layer = - TRT_ENGINE_ADD_LAYER(engine_, Slice, *shape_layer_itensor, start_dim, - size_dim, stride_dim); - uint32_t reduce_dim = 1; - auto* reduce_prod_layer = TRT_ENGINE_ADD_LAYER( - engine_, Reduce, *(slice_layer->getOutput(0)), - nvinfer1::ReduceOperation::kPROD, reduce_dim, true); - - nvinfer1::ITensor* input_shape = nullptr; - if (start_axis == 0 && stop_axis == dims - 1) { - input_shape = reduce_prod_layer->getOutput(0); - } else { - std::vector itensors; - if (start_axis > 0) { - nvinfer1::Dims left_start_dim, left_size_dim, left_stride_dim; - left_start_dim.nbDims = 1; - left_size_dim.nbDims = 1; - left_stride_dim.nbDims = 1; - left_start_dim.d[0] = 0; - left_size_dim.d[0] = start_axis; - left_stride_dim.d[0] = 1; - auto* slice_layer_left = TRT_ENGINE_ADD_LAYER( - engine_, Slice, *shape_layer_itensor, left_start_dim, - left_size_dim, left_stride_dim); - itensors.push_back(slice_layer_left->getOutput(0)); + int dim_prod = 1; + int dim_negative = 0; + nvinfer1::Dims flatten_dim; + flatten_dim.nbDims = dims - (stop_axis - start_axis); + bool need_slice = false; + for (int i = 0, j = 0; i < dims; ++i) { + int dim_i = input_dim.d[i]; + if (start_axis <= i && i <= stop_axis) { + if (dim_i < 0) { + need_slice = true; + break; + } + dim_prod *= dim_i; + if (i == stop_axis) { + flatten_dim.d[j++] = dim_prod; + } + } else { + if (dim_i < 0) dim_negative++; + if (dim_negative > 1) { + need_slice = true; + break; + } + flatten_dim.d[j++] = input_dim.d[i]; } - itensors.push_back(reduce_prod_layer->getOutput(0)); - if (stop_axis < dims - 1) { - nvinfer1::Dims right_start_dim, right_size_dim, right_stride_dim; - right_start_dim.nbDims = 1; - right_size_dim.nbDims = 1; - right_stride_dim.nbDims = 1; - right_start_dim.d[0] = stop_axis + 1; - right_size_dim.d[0] = dims - stop_axis - 1; - right_stride_dim.d[0] = 1; - auto* slice_layer_right = TRT_ENGINE_ADD_LAYER( - engine_, Slice, *shape_layer_itensor, right_start_dim, - right_size_dim, right_stride_dim); - itensors.push_back(slice_layer_right->getOutput(0)); + } + + if (need_slice) { + VLOG(3) << "slice input dim when the input dimension has -1"; + auto* shape_layer = TRT_ENGINE_ADD_LAYER(engine_, Shape, *input); + auto* shape_layer_itensor = shape_layer->getOutput(0); + + nvinfer1::Dims start_dim, size_dim, stride_dim; + start_dim.nbDims = 1; + size_dim.nbDims = 1; + stride_dim.nbDims = 1; + start_dim.d[0] = start_axis; + size_dim.d[0] = stop_axis - start_axis + 1; + stride_dim.d[0] = 1; + auto* slice_layer = + TRT_ENGINE_ADD_LAYER(engine_, Slice, *shape_layer_itensor, + start_dim, size_dim, stride_dim); + uint32_t reduce_dim = 1; + auto* reduce_prod_layer = TRT_ENGINE_ADD_LAYER( + engine_, Reduce, *(slice_layer->getOutput(0)), + nvinfer1::ReduceOperation::kPROD, reduce_dim, true); + + nvinfer1::ITensor* input_shape = nullptr; + if (start_axis == 0 && stop_axis == dims - 1) { + input_shape = reduce_prod_layer->getOutput(0); + } else { + std::vector itensors; + if (start_axis > 0) { + nvinfer1::Dims left_start_dim, left_size_dim, left_stride_dim; + left_start_dim.nbDims = 1; + left_size_dim.nbDims = 1; + left_stride_dim.nbDims = 1; + left_start_dim.d[0] = 0; + left_size_dim.d[0] = start_axis; + left_stride_dim.d[0] = 1; + auto* slice_layer_left = TRT_ENGINE_ADD_LAYER( + engine_, Slice, *shape_layer_itensor, left_start_dim, + left_size_dim, left_stride_dim); + itensors.push_back(slice_layer_left->getOutput(0)); + } + itensors.push_back(reduce_prod_layer->getOutput(0)); + if (stop_axis < dims - 1) { + nvinfer1::Dims right_start_dim, right_size_dim, right_stride_dim; + right_start_dim.nbDims = 1; + right_size_dim.nbDims = 1; + right_stride_dim.nbDims = 1; + right_start_dim.d[0] = stop_axis + 1; + right_size_dim.d[0] = dims - stop_axis - 1; + right_stride_dim.d[0] = 1; + auto* slice_layer_right = TRT_ENGINE_ADD_LAYER( + engine_, Slice, *shape_layer_itensor, right_start_dim, + right_size_dim, right_stride_dim); + itensors.push_back(slice_layer_right->getOutput(0)); + } + auto* concat_layer = TRT_ENGINE_ADD_LAYER( + engine_, Concatenation, itensors.data(), itensors.size()); + concat_layer->setAxis(0); + input_shape = concat_layer->getOutput(0); } - auto* concat_layer = TRT_ENGINE_ADD_LAYER( - engine_, Concatenation, itensors.data(), itensors.size()); - concat_layer->setAxis(0); - input_shape = concat_layer->getOutput(0); + layer->setInput(1, *input_shape); + } else { + layer->setReshapeDimensions(flatten_dim); } - layer = TRT_ENGINE_ADD_LAYER(engine_, Shuffle, *input); - layer->setInput(1, *input_shape); } + auto output_name = op_desc.Output("Out")[0]; RreplenishLayerAndOutput(layer, "flatten_contiguous_range", {output_name}, test_mode); diff --git a/paddle/fluid/inference/tests/api/CMakeLists.txt b/paddle/fluid/inference/tests/api/CMakeLists.txt index e9b8c0ce70f66..fc85f83661889 100644 --- a/paddle/fluid/inference/tests/api/CMakeLists.txt +++ b/paddle/fluid/inference/tests/api/CMakeLists.txt @@ -346,17 +346,13 @@ inference_analysis_test(test_analyzer_transformer_profile SRCS analyzer_transfor --cpu_num_threads=${CPU_NUM_THREADS_ON_CI}) # VIT-OCR -set(VIT_OCR_URL "https://paddle-qa.bj.bcebos.com/inference_model/2.1.1/ocr") -set(VIT_OCR_INSTALL_DIR "${INFERENCE_DEMO_INSTALL_DIR}/vit_ocr") +set(VIT_OCR_INSTALL_DIR "${INFERENCE_DEMO_INSTALL_DIR}/vit") if (NOT EXISTS ${VIT_OCR_INSTALL_DIR}/vit_ocr.tgz) - inference_download_and_uncompress_without_verify(${VIT_OCR_INSTALL_DIR} ${VIT_OCR_URL} vit_ocr.tgz) -endif() -if (NOT EXISTS ${VIT_OCR_INSTALL_DIR}/datavit.txt) - file(DOWNLOAD ${VIT_OCR_URL}/datavit.txt ${VIT_OCR_INSTALL_DIR}/datavit.txt) + inference_download_and_uncompress_without_verify(${VIT_OCR_INSTALL_DIR} ${INFERENCE_URL} "ocr/vit_ocr.tgz") endif() inference_analysis_test(test_analyzer_vit_ocr SRCS analyzer_vit_ocr_tester.cc EXTRA_DEPS ${INFERENCE_EXTRA_DEPS} - ARGS --infer_model=${VIT_OCR_INSTALL_DIR}/vit_ocr --infer_data=${VIT_OCR_INSTALL_DIR}/datavit.txt) + ARGS --infer_model=${VIT_OCR_INSTALL_DIR}/vit_ocr/model --infer_data=${VIT_OCR_INSTALL_DIR}/vit_ocr/datavit.txt) # ocr set(OCR_INSTALL_DIR "${INFERENCE_DEMO_INSTALL_DIR}/ocr") diff --git a/paddle/fluid/inference/tests/infer_ut/test_det_mv3_db.cc b/paddle/fluid/inference/tests/infer_ut/test_det_mv3_db.cc index eb31acbdf7ca1..115ce0bbb4d00 100644 --- a/paddle/fluid/inference/tests/infer_ut/test_det_mv3_db.cc +++ b/paddle/fluid/inference/tests/infer_ut/test_det_mv3_db.cc @@ -35,11 +35,26 @@ paddle::test::Record PrepareInput(int batch_size, int image_shape = 640) { void PrepareDynamicShape(paddle_infer::Config* config, int max_batch_size = 4) { // set dynamic shape range std::map> min_input_shape = { - {"x", {1, 3, 50, 50}}}; + {"x", {1, 3, 224, 224}}, + {"conv2d_124.tmp_0", {1, 256, 56, 56}}, + {"nearest_interp_v2_2.tmp_0", {1, 256, 56, 56}}, + {"nearest_interp_v2_3.tmp_0", {1, 64, 56, 56}}, + {"nearest_interp_v2_4.tmp_0", {1, 64, 56, 56}}, + {"nearest_interp_v2_5.tmp_0", {1, 64, 56, 56}}}; std::map> max_input_shape = { - {"x", {max_batch_size, 3, 1600, 1600}}}; + {"x", {max_batch_size, 3, 448, 448}}, + {"conv2d_124.tmp_0", {max_batch_size, 256, 112, 112}}, + {"nearest_interp_v2_2.tmp_0", {max_batch_size, 256, 112, 112}}, + {"nearest_interp_v2_3.tmp_0", {max_batch_size, 64, 112, 112}}, + {"nearest_interp_v2_4.tmp_0", {max_batch_size, 64, 112, 112}}, + {"nearest_interp_v2_5.tmp_0", {max_batch_size, 64, 112, 112}}}; std::map> opt_input_shape = { - {"x", {1, 3, 640, 640}}}; + {"x", {1, 3, 256, 256}}, + {"conv2d_124.tmp_0", {1, 256, 64, 64}}, + {"nearest_interp_v2_2.tmp_0", {1, 256, 64, 64}}, + {"nearest_interp_v2_3.tmp_0", {1, 64, 64, 64}}, + {"nearest_interp_v2_4.tmp_0", {1, 64, 64, 64}}, + {"nearest_interp_v2_5.tmp_0", {1, 64, 64, 64}}}; config->SetTRTDynamicShapeInfo(min_input_shape, max_input_shape, opt_input_shape); } @@ -76,7 +91,7 @@ TEST(tensorrt_tester_det_mv3_db, multi_thread2_trt_fp32_dynamic_shape_bz2) { int thread_num = 2; // thread > 2 may OOM // init input data std::map my_input_data_map; - my_input_data_map["x"] = PrepareInput(2, 640); + my_input_data_map["x"] = PrepareInput(2, 256); // init output data std::map infer_output_data, truth_output_data; @@ -90,7 +105,7 @@ TEST(tensorrt_tester_det_mv3_db, multi_thread2_trt_fp32_dynamic_shape_bz2) { FLAGS_modeldir + "/inference.pdiparams"); config.EnableUseGpu(100, 0); config.EnableTensorRtEngine( - 1 << 20, 2, 3, paddle_infer::PrecisionType::kFloat32, false, false); + 1 << 20, 4, 3, paddle_infer::PrecisionType::kFloat32, false, false); PrepareDynamicShape(&config, 4); // get groudtruth by disbale ir paddle_infer::services::PredictorPool pred_pool_no_ir(config_no_ir, 1); diff --git a/paddle/fluid/inference/tests/infer_ut/test_ppyolo_mbv3.cc b/paddle/fluid/inference/tests/infer_ut/test_ppyolo_mbv3.cc index ff1647432a12d..eb8c5bedc0375 100644 --- a/paddle/fluid/inference/tests/infer_ut/test_ppyolo_mbv3.cc +++ b/paddle/fluid/inference/tests/infer_ut/test_ppyolo_mbv3.cc @@ -93,7 +93,7 @@ TEST(tensorrt_tester_ppyolo_mbv3, multi_thread4_trt_fp32_bz2) { for (int i = 0; i < thread_num; ++i) { LOG(INFO) << "join tid : " << i; threads[i].join(); - CompareRecord(&truth_output_data, &infer_output_data, 1e-2); + CompareRecord(&truth_output_data, &infer_output_data, 0.18); // TODO(OliverLPH): precision set to 1e-2 since input is fake, change to // real input later } diff --git a/paddle/fluid/inference/tests/infer_ut/test_resnet50.cc b/paddle/fluid/inference/tests/infer_ut/test_resnet50.cc index 01bec2916e94a..28623bc89a065 100644 --- a/paddle/fluid/inference/tests/infer_ut/test_resnet50.cc +++ b/paddle/fluid/inference/tests/infer_ut/test_resnet50.cc @@ -87,7 +87,7 @@ TEST(tensorrt_tester_resnet50, trt_fp32_bz2) { SingleThreadPrediction(pred_pool.Retrive(0), &my_input_data_map, &infer_output_data); // check outputs - CompareRecord(&truth_output_data, &infer_output_data); + CompareRecord(&truth_output_data, &infer_output_data, 2e-4); std::cout << "finish test" << std::endl; } @@ -122,7 +122,7 @@ TEST(tensorrt_tester_resnet50, serial_diff_batch_trt_fp32) { SingleThreadPrediction(pred_pool.Retrive(0), &my_input_data_map, &infer_output_data); // check outputs - CompareRecord(&truth_output_data, &infer_output_data); + CompareRecord(&truth_output_data, &infer_output_data, 1e-4); } std::cout << "finish test" << std::endl; } @@ -164,7 +164,7 @@ TEST(tensorrt_tester_resnet50, multi_thread4_trt_fp32_bz2) { for (int i = 0; i < thread_num; ++i) { LOG(INFO) << "join tid : " << i; threads[i].join(); - CompareRecord(&truth_output_data, &infer_output_data); + CompareRecord(&truth_output_data, &infer_output_data, 2e-4); } std::cout << "finish multi-thread test" << std::endl; diff --git a/paddle/fluid/inference/utils/table_printer.cc b/paddle/fluid/inference/utils/table_printer.cc index bd19320cbe647..628465c423b03 100644 --- a/paddle/fluid/inference/utils/table_printer.cc +++ b/paddle/fluid/inference/utils/table_printer.cc @@ -53,7 +53,7 @@ std::string TablePrinter::PrintTable() { AddRowDivider(ss); - return std::move(ss.str()); + return ss.str(); } TablePrinter::TablePrinter(const std::vector& header) { diff --git a/paddle/fluid/memory/allocation/allocator_facade.cc b/paddle/fluid/memory/allocation/allocator_facade.cc index e2730a1b825e9..35ad27f4c62b5 100644 --- a/paddle/fluid/memory/allocation/allocator_facade.cc +++ b/paddle/fluid/memory/allocation/allocator_facade.cc @@ -415,6 +415,23 @@ class AllocatorFacadePrivate { void SetDefaultStream(const platform::CUDAPlace& place, gpuStream_t stream) { const std::shared_ptr& allocator = GetDefaultStreamSafeCUDAAllocator(place); + + // NOTE(Ruibiao): The default stream will be set when the CUDADeviceContext + // created. Normally, the DeviceContextPool is a global singleton and one + // Place only correspond to one DeviceContext. However, to support + // multi-stream scheduling, standalone executor creates two extra + // DeviceContextPools for H2D and D2H stream in StreamAnalyzer, which make + // one Place correspond to multiple DeviceContext and unexpectedly reset the + // default stream in runtime. To avoid this behavior, we do not allow + // changing default stream after initially setting. + if (allocator->GetDefaultStream() != nullptr) { + VLOG(5) << "The default stream for StreamSafeCUDAAllocator(" + << allocator.get() << ") in " << place << " has been set to " + << allocator->GetDefaultStream() + << " before, not allow to change now."; + return; + } + allocator->SetDefaultStream(stream); VLOG(8) << "Set default stream to " << stream << " for StreamSafeCUDAAllocator(" << allocator.get() << ") in " @@ -818,6 +835,16 @@ class AllocatorFacadePrivate { platform::MLUPlace p(i); system_allocators_[p] = std::make_shared(p); } +#endif +#ifdef PADDLE_WITH_CUSTOM_DEVICE + auto device_types = phi::DeviceManager::GetAllCustomDeviceTypes(); + for (const auto& dev_type : device_types) { + for (size_t dev_id = 0; + dev_id < phi::DeviceManager::GetDeviceCount(dev_type); dev_id++) { + platform::CustomPlace p(dev_type, dev_id); + system_allocators_[p] = std::make_shared(p); + } + } #endif } diff --git a/paddle/fluid/operators/amp/update_loss_scaling_op.cc b/paddle/fluid/operators/amp/update_loss_scaling_op.cc index b974f606720b2..8354650df0237 100644 --- a/paddle/fluid/operators/amp/update_loss_scaling_op.cc +++ b/paddle/fluid/operators/amp/update_loss_scaling_op.cc @@ -68,6 +68,18 @@ class UpdateLossScalingOp : public framework::OperatorWithKernel { return framework::OpKernelType(dtype, ctx.GetPlace()); } + + framework::OpKernelType GetKernelTypeForVar( + const std::string& var_name, const framework::Tensor& tensor, + const framework::OpKernelType& expected_kernel_type) const override { +#ifndef PADDLE_WITH_XPU + if (var_name == "FoundInfinite" || var_name == "StopUpdate") { + return expected_kernel_type; + } +#endif + return framework::OperatorWithKernel::GetKernelTypeForVar( + var_name, tensor, expected_kernel_type); + } }; class UpdateLossScalingOpMaker : public framework::OpProtoAndCheckerMaker { @@ -93,6 +105,10 @@ class UpdateLossScalingOpMaker : public framework::OpProtoAndCheckerMaker { AddOutput("LossScaling", "(Tensor) 1-dim tensor, updated loss scaling."); AddOutput("OutGoodSteps", "(Tensor) 1-dim tensor, pdated good steps."); AddOutput("OutBadSteps", "(Tensor) 1-dim tensor, updated bad steps."); + AddOutput("StopUpdate", + "(Tensor) 1-dim tensor. Stop updating loss scaling, and just " + "zero inputs. It has higher priority than Attr(stop_update).") + .AsDispensable(); AddAttr("incr_every_n_steps", "A value represents increasing loss scaling every n " "consecutive steps with finite gradients."); @@ -131,8 +147,8 @@ decr_every_n_nan_or_inf steps and each step some gradients are infinite. } }; -template -class UpdateLossScalingFunctor { +template +class UpdateLossScalingFunctor { public: void operator()(const platform::CPUDeviceContext& ctx, const bool* found_inf_data, const T* pre_loss_scaling_data, @@ -141,6 +157,10 @@ class UpdateLossScalingFunctor { const int decr_every_n_nan_or_inf, const float incr_ratio, const float decr_ratio, T* updated_loss_scaling_data, int* good_out_data, int* bad_out_data) const { + PADDLE_ENFORCE_EQ( + IsFoundInfOnCPU, true, + platform::errors::InvalidArgument( + "The Input(FoundInfinite) should be on the CPUPlace.")); Update(found_inf_data, pre_loss_scaling_data, good_in_data, bad_in_data, incr_every_n_steps, decr_every_n_nan_or_inf, incr_ratio, decr_ratio, updated_loss_scaling_data, good_out_data, diff --git a/paddle/fluid/operators/amp/update_loss_scaling_op.cu b/paddle/fluid/operators/amp/update_loss_scaling_op.cu index 6d9cd96a3fb9a..43f8f84578c70 100644 --- a/paddle/fluid/operators/amp/update_loss_scaling_op.cu +++ b/paddle/fluid/operators/amp/update_loss_scaling_op.cu @@ -21,9 +21,9 @@ limitations under the License. */ namespace paddle { namespace operators { -template +template __global__ void GpuUpdateLossScaling( - const bool* found_inf_data, const T* pre_loss_scaling_data, + const FoundNanInfFlagT found_inf_data, const T* pre_loss_scaling_data, const int* good_in_data, const int* bad_in_data, const int incr_every_n_steps, const int decr_every_n_nan_or_inf, const float incr_ratio, const float decr_ratio, @@ -70,8 +70,9 @@ __global__ void FusedFillIf(T** outs, const size_t xs_size, } } -template -class UpdateLossScalingFunctor { +template +class UpdateLossScalingFunctor { public: void operator()(const platform::CUDADeviceContext& dev_ctx, const bool* found_inf_data, const T* pre_loss_scaling_data, @@ -80,10 +81,17 @@ class UpdateLossScalingFunctor { const int decr_every_n_nan_or_inf, const float incr_ratio, const float decr_ratio, T* updated_loss_scaling_data, int* good_out_data, int* bad_out_data) const { - GpuUpdateLossScaling<<<1, 1, 0, dev_ctx.stream()>>>( - found_inf_data, pre_loss_scaling_data, good_in_data, bad_in_data, - incr_every_n_steps, decr_every_n_nan_or_inf, incr_ratio, decr_ratio, - updated_loss_scaling_data, good_out_data, bad_out_data); + if (IsFoundInfOnCPU) { + GpuUpdateLossScaling<<<1, 1, 0, dev_ctx.stream()>>>( + *found_inf_data, pre_loss_scaling_data, good_in_data, bad_in_data, + incr_every_n_steps, decr_every_n_nan_or_inf, incr_ratio, decr_ratio, + updated_loss_scaling_data, good_out_data, bad_out_data); + } else { + GpuUpdateLossScaling<<<1, 1, 0, dev_ctx.stream()>>>( + found_inf_data, pre_loss_scaling_data, good_in_data, bad_in_data, + incr_every_n_steps, decr_every_n_nan_or_inf, incr_ratio, decr_ratio, + updated_loss_scaling_data, good_out_data, bad_out_data); + } } }; diff --git a/paddle/fluid/operators/amp/update_loss_scaling_op.h b/paddle/fluid/operators/amp/update_loss_scaling_op.h index d6eddd36a4551..41eb94247f593 100644 --- a/paddle/fluid/operators/amp/update_loss_scaling_op.h +++ b/paddle/fluid/operators/amp/update_loss_scaling_op.h @@ -25,6 +25,7 @@ #include "paddle/fluid/platform/enforce.h" #include "paddle/fluid/platform/errors.h" #include "paddle/phi/core/hostdevice.h" +#include "paddle/phi/kernels/funcs/math_function.h" namespace paddle { namespace operators { @@ -40,8 +41,16 @@ inline HOSTDEVICE bool check_finite(T value) { #endif } -template -inline HOSTDEVICE void Update(const bool* found_inf_data, +inline HOSTDEVICE bool IsFoundNanInf(const bool found_nan_inf_data) { + return found_nan_inf_data; +} + +inline HOSTDEVICE bool IsFoundNanInf(const bool* found_nan_inf_data) { + return *found_nan_inf_data; +} + +template +inline HOSTDEVICE void Update(const FoundInfFlagT found_inf_data, const T* pre_loss_scaling_data, const int* good_in_data, const int* bad_in_data, const int incr_every_n_steps, @@ -49,7 +58,7 @@ inline HOSTDEVICE void Update(const bool* found_inf_data, const float incr_ratio, const float decr_ratio, T* updated_loss_scaling_data, int* good_out_data, int* bad_out_data) { - if (*found_inf_data) { + if (IsFoundNanInf(found_inf_data)) { *good_out_data = 0; *bad_out_data = *bad_in_data + 1; if (*bad_out_data == decr_every_n_nan_or_inf) { @@ -72,7 +81,7 @@ inline HOSTDEVICE void Update(const bool* found_inf_data, } } -template +template class UpdateLossScalingFunctor { public: void operator()(const DeviceContext& dev_ctx, const bool* found_inf_data, @@ -106,9 +115,33 @@ class UpdateLossScalingKernel : public framework::OpKernel { platform::errors::InvalidArgument( "FoundInfinite must has only one element.")); const bool* found_inf_data = found_inf->data(); + bool is_found_inf_on_cpu = platform::is_cpu_place(found_inf->place()); + + if (is_found_inf_on_cpu) { + if (*found_inf_data) { + phi::funcs::SetConstant set_constant; + for (auto* out : outs) { + out->mutable_data(dev_ctx.GetPlace()); + set_constant(dev_ctx, out, static_cast(0)); + } + } + } else { + LazyZeros{}(dev_ctx, found_inf_data, xs, outs); + } - LazyZeros{}(dev_ctx, found_inf_data, xs, outs); - const bool stop_update = ctx.Attr("stop_update"); + const auto* stop_update_tensor = ctx.Input("StopUpdate"); + bool stop_update = false; + if (stop_update_tensor && stop_update_tensor->IsInitialized()) { + if (platform::is_cpu_place(stop_update_tensor->place())) { + stop_update = stop_update_tensor->data()[0]; + } else { + framework::Tensor tmp_tensor; + framework::TensorCopySync(*stop_update_tensor, platform::CPUPlace(), + &tmp_tensor); + stop_update = tmp_tensor.data()[0]; + } + } + stop_update |= ctx.Attr("stop_update"); if (stop_update) { return; } @@ -133,10 +166,17 @@ class UpdateLossScalingKernel : public framework::OpKernel { ctx.Attr("decr_every_n_nan_or_inf"); const float incr_ratio = ctx.Attr("incr_ratio"); const float decr_ratio = ctx.Attr("decr_ratio"); - UpdateLossScalingFunctor{}( - dev_ctx, found_inf_data, pre_loss_scaling_data, good_in_data, - bad_in_data, incr_every_n_steps, decr_every_n_nan_or_inf, incr_ratio, - decr_ratio, updated_loss_scaling_data, good_out_data, bad_out_data); + if (is_found_inf_on_cpu) { + UpdateLossScalingFunctor{}( + dev_ctx, found_inf_data, pre_loss_scaling_data, good_in_data, + bad_in_data, incr_every_n_steps, decr_every_n_nan_or_inf, incr_ratio, + decr_ratio, updated_loss_scaling_data, good_out_data, bad_out_data); + } else { + UpdateLossScalingFunctor{}( + dev_ctx, found_inf_data, pre_loss_scaling_data, good_in_data, + bad_in_data, incr_every_n_steps, decr_every_n_nan_or_inf, incr_ratio, + decr_ratio, updated_loss_scaling_data, good_out_data, bad_out_data); + } } }; diff --git a/paddle/fluid/operators/amp/update_loss_scaling_op_npu.cc b/paddle/fluid/operators/amp/update_loss_scaling_op_npu.cc index 1393da7dd57a7..5808841333f08 100644 --- a/paddle/fluid/operators/amp/update_loss_scaling_op_npu.cc +++ b/paddle/fluid/operators/amp/update_loss_scaling_op_npu.cc @@ -131,7 +131,8 @@ void Update(const platform::NPUDeviceContext& ctx, } template -class UpdateLossScalingFunctor { +class UpdateLossScalingFunctor { public: void operator()(const platform::NPUDeviceContext& dev_ctx, const std::vector found_inf_vec, @@ -236,7 +237,7 @@ class UpdateLossScalingNPUKernel : public framework::OpKernel { ctx.Attr("decr_every_n_nan_or_inf"); const float incr_ratio = ctx.Attr("incr_ratio"); const float decr_ratio = ctx.Attr("decr_ratio"); - UpdateLossScalingFunctor{}( + UpdateLossScalingFunctor{}( dev_ctx, found_inf_vec, pre_loss_scaling, good_in, bad_in, incr_every_n_steps, decr_every_n_nan_or_inf, incr_ratio, decr_ratio, updated_loss_scaling, good_out, bad_out); diff --git a/paddle/fluid/operators/batch_norm_op_xpu.cc b/paddle/fluid/operators/batch_norm_op_xpu.cc index da138fb482e5a..0893324c602a8 100644 --- a/paddle/fluid/operators/batch_norm_op_xpu.cc +++ b/paddle/fluid/operators/batch_norm_op_xpu.cc @@ -53,8 +53,12 @@ class BatchNormXPUKernel : public framework::OpKernel { "But received: the size of input's dimensions is [%d]", x_dims.size())); - int N, C, H, W, D; + int N = -1, C = -1, H = -1, W = -1, D = -1; ExtractNCWHD(x_dims, data_layout, &N, &C, &H, &W, &D); + N = (N == 0) ? 1 : N; + C = (C == 0) ? 1 : C; + H = (H == 0) ? 1 : H; + W = (W == 0) ? 1 : W; const auto *scale = ctx.Input("Scale"); const auto *bias = ctx.Input("Bias"); @@ -103,12 +107,6 @@ class BatchNormXPUKernel : public framework::OpKernel { "The batch_norm XPU API return wrong value[%d %s]", r, XPUAPIErrorMsg[r])); } else { - PADDLE_ENFORCE_EQ( - data_layout_str == "NCHW", true, - platform::errors::InvalidArgument( - "The batch_norm_infer 'data_layout' attribute must be NCHW. " - "But recevived 'data_layout' is [%s].", - data_layout_str)); const auto *mean = ctx.Input("Mean"); const auto *variance = ctx.Input("Variance"); const auto *mean_data = mean->data(); @@ -222,8 +220,12 @@ class BatchNormGradXPUKernel : public framework::OpKernel { "But received: the size of input's dimensions is [%d]", x_dims.size())); - int N, C, H, W, D; + int N = -1, C = -1, H = -1, W = -1, D = -1; ExtractNCWHD(x_dims, data_layout, &N, &C, &H, &W, &D); + N = (N == 0) ? 1 : N; + C = (C == 0) ? 1 : C; + H = (H == 0) ? 1 : H; + W = (W == 0) ? 1 : W; const auto *x_data = x->data(); const auto *d_y_data = d_y->data(); diff --git a/paddle/fluid/operators/channel_shuffle_op.cc b/paddle/fluid/operators/channel_shuffle_op.cc new file mode 100644 index 0000000000000..74b2e04e63f70 --- /dev/null +++ b/paddle/fluid/operators/channel_shuffle_op.cc @@ -0,0 +1,100 @@ +// Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "paddle/fluid/framework/infershape_utils.h" +#include "paddle/fluid/framework/op_registry.h" +#include "paddle/fluid/framework/op_version_registry.h" +#include "paddle/phi/core/infermeta_utils.h" +#include "paddle/phi/infermeta/backward.h" +#include "paddle/phi/infermeta/unary.h" + +namespace paddle { +namespace operators { + +class ChannelShuffleOp : public framework::OperatorWithKernel { + public: + using framework::OperatorWithKernel::OperatorWithKernel; +}; + +class ChannelShuffleOpMaker : public framework::OpProtoAndCheckerMaker { + public: + void Make() override { + AddInput("X", + "(Tensor, default Tensor), " + "the input feature data of ChannelShuffleOp, the layout is " + "[N, C, H, W] or [N, H, W, C]."); + AddOutput("Out", + "(Tensor, default Tensor), the output of " + "ChannelShuffleOp. The layout is also [N, C, " + "H, W] or [N, H, W, C]."); + AddAttr("groups", "number of groups to divide channels in."); + AddAttr( + "data_format", + "An optional string from: \"NHWC\", \"NCHW\". " + "Defaults to \"NHWC\", Specify the data format of the input data.") + .SetDefault("NCHW"); + + AddComment(R"DOC( + Channel Shuffle operator + This operator divides channels in a tensor of shape :math:`(*, C, H, W)` + into :math:`g` groups and rearranges them as :math:`(*, C/g, g, H, W)` + while keeping the original tensor shape. + + Please refer to the paper: + `ShuffleNet: An Extremely Efficient Convolutional Neural Network for + Mobile Devices `_ + by Zhang et. al (2017) for more details. + + )DOC"); + } +}; + +class ChannelShuffleGradOp : public framework::OperatorWithKernel { + public: + using framework::OperatorWithKernel::OperatorWithKernel; +}; + +template +class ChannelShuffleGradOpMaker : public framework::SingleGradOpMaker { + public: + using framework::SingleGradOpMaker::SingleGradOpMaker; + + protected: + void Apply(GradOpPtr op) const override { + op->SetType("channel_shuffle_grad"); + op->SetInput(framework::GradVarName("Out"), this->OutputGrad("Out")); + op->SetAttrMap(this->Attrs()); + op->SetOutput(framework::GradVarName("X"), this->InputGrad("X")); + } +}; + +} // namespace operators +} // namespace paddle + +namespace ops = paddle::operators; +DECLARE_INFER_SHAPE_FUNCTOR(channel_shuffle, ChannelShuffleInferShapeFunctor, + PD_INFER_META(phi::ChannelShuffleInferMeta)); + +REGISTER_OPERATOR(channel_shuffle, ops::ChannelShuffleOp, + ops::ChannelShuffleOpMaker, + ops::ChannelShuffleGradOpMaker, + ops::ChannelShuffleGradOpMaker, + ChannelShuffleInferShapeFunctor); + +DECLARE_INFER_SHAPE_FUNCTOR(channel_shuffle_grad, + ChannelShuffleGradInferShapeFunctor, + PD_INFER_META(phi::ChannelShuffleGradInferMeta)); + +REGISTER_OPERATOR(channel_shuffle_grad, ops::ChannelShuffleGradOp, + ChannelShuffleGradInferShapeFunctor); diff --git a/paddle/fluid/operators/collective/c_comm_init_op.cc b/paddle/fluid/operators/collective/c_comm_init_op.cc index 39acb50d4e870..82d3b1b1dbfea 100644 --- a/paddle/fluid/operators/collective/c_comm_init_op.cc +++ b/paddle/fluid/operators/collective/c_comm_init_op.cc @@ -83,7 +83,6 @@ class CCommInitOp : public framework::OperatorBase { UniqueId* comm_id = var->GetMutable(); int nranks = Attr("nranks"); - int rank_id = Attr("rank"); int rid = Attr("ring_id"); #if defined(PADDLE_WITH_XPU_BKCL) @@ -98,8 +97,18 @@ class CCommInitOp : public framework::OperatorBase { if (Attr("device_id") >= 0) { device_id = Attr("device_id"); } + +#if defined(PADDLE_WITH_XPU_BKCL) && defined(PADDLE_WITH_HETERPS) && \ + defined(PADDLE_WITH_PSLIB) + // XPUPS rank_id only equals 0, so replace rank_id with device_id + CommContext::Instance().CreateComm(comm_id, nranks, device_id, device_id, + rid); +#else + int rank_id = Attr("rank"); CommContext::Instance().CreateComm(comm_id, nranks, rank_id, device_id, rid); +#endif + #endif } }; diff --git a/paddle/fluid/operators/collective/c_sync_calc_stream_op.cc b/paddle/fluid/operators/collective/c_sync_calc_stream_op.cc index 42584948e0651..088366dbc8f69 100644 --- a/paddle/fluid/operators/collective/c_sync_calc_stream_op.cc +++ b/paddle/fluid/operators/collective/c_sync_calc_stream_op.cc @@ -76,7 +76,15 @@ class CSyncCalcStreamKernel : public framework::OpKernel { auto dev_ctx = static_cast( platform::DeviceContextPool::Instance().Get(place)); platform::MLUStreamSync(dev_ctx->stream()); +#elif defined(PADDLE_WITH_XPU_BKCL) + auto place = ctx.GetPlace(); + PADDLE_ENFORCE_EQ(platform::is_xpu_place(place), true, + platform::errors::PreconditionNotMet( + "Sync stream op can run on xpu place only for now.")); + auto dev_ctx = static_cast( + platform::DeviceContextPool::Instance().Get(place)); + dev_ctx->Wait(); #else PADDLE_THROW(platform::errors::PreconditionNotMet( "PaddlePaddle should compile with GPU.")); @@ -97,3 +105,5 @@ REGISTER_OP_CUDA_KERNEL(c_sync_calc_stream, ops::CSyncCalcStreamKernel); REGISTER_OP_NPU_KERNEL(c_sync_calc_stream, ops::CSyncCalcStreamKernel); REGISTER_OP_MLU_KERNEL(c_sync_calc_stream, ops::CSyncCalcStreamKernel); + +REGISTER_OP_XPU_KERNEL(c_sync_calc_stream, ops::CSyncCalcStreamKernel); diff --git a/paddle/fluid/operators/collective/c_sync_comm_stream_op.cc b/paddle/fluid/operators/collective/c_sync_comm_stream_op.cc index 37ce4ef7ee21d..5a9a00aa8e4d2 100644 --- a/paddle/fluid/operators/collective/c_sync_comm_stream_op.cc +++ b/paddle/fluid/operators/collective/c_sync_comm_stream_op.cc @@ -20,7 +20,6 @@ limitations under the License. */ #endif #if defined(PADDLE_WITH_ASCEND_CL) -#include "paddle/fluid/platform/collective_helper.h" #include "paddle/fluid/platform/device/npu/hccl_helper.h" #endif @@ -28,6 +27,10 @@ limitations under the License. */ #include "paddle/fluid/platform/device/mlu/cncl_helper.h" #endif +#if defined(PADDLE_WITH_XPU_BKCL) || defined(PADDLE_WITH_ASCEND_CL) +#include "paddle/fluid/platform/collective_helper.h" +#endif + namespace paddle { namespace operators { @@ -94,7 +97,16 @@ class CSyncCommStreamKernel : public framework::OpKernel { auto stream = platform::CNCLCommContext::Instance().Get(ring_id, place)->stream(); platform::MLUStreamSync(stream); - +#elif defined(PADDLE_WITH_XPU_BKCL) + auto place = ctx.GetPlace(); + PADDLE_ENFORCE_EQ(platform::is_xpu_place(place), true, + platform::errors::PreconditionNotMet( + "Sync stream op can run on xpu place only for now.")); + int ring_id = ctx.Attr("ring_id"); + auto comm_dev_ctx = platform::BKCLCommContext::Instance() + .Get(ring_id, place) + ->dev_context(); + comm_dev_ctx->Wait(); #else PADDLE_THROW(platform::errors::PreconditionNotMet( "PaddlePaddle should compile with GPU.")); @@ -115,3 +127,5 @@ REGISTER_OP_CUDA_KERNEL(c_sync_comm_stream, ops::CSyncCommStreamKernel); REGISTER_OP_NPU_KERNEL(c_sync_comm_stream, ops::CSyncCommStreamKernel); REGISTER_OP_MLU_KERNEL(c_sync_comm_stream, ops::CSyncCommStreamKernel); + +REGISTER_OP_XPU_KERNEL(c_sync_comm_stream, ops::CSyncCommStreamKernel); diff --git a/paddle/fluid/operators/conv_op_xpu.cc b/paddle/fluid/operators/conv_op_xpu.cc index e4751f1f26008..cc5c20d392809 100644 --- a/paddle/fluid/operators/conv_op_xpu.cc +++ b/paddle/fluid/operators/conv_op_xpu.cc @@ -38,9 +38,10 @@ class GemmConvXPUKernel : public framework::OpKernel { const std::string padding_algorithm = context.Attr("padding_algorithm"); - PADDLE_ENFORCE_EQ(data_format == "NHWC" || data_format == "NDHWC", false, - platform::errors::InvalidArgument( - ("XPU do support data_format is NCHW in conv op."))); + PADDLE_ENFORCE_EQ( + data_format == "NDHWC", false, + platform::errors::InvalidArgument( + ("XPU does not support data_format is NDHWC in conv op."))); framework::DDim in_data_dims = phi::slice_ddim(input->dims(), 2, input->dims().size()); @@ -50,11 +51,18 @@ class GemmConvXPUKernel : public framework::OpKernel { UpdatePaddingAndDilation(&paddings, &dilations, padding_algorithm, in_data_dims, strides, ksize); - const int batch_size = static_cast(input->dims()[0]); - const int img_c = static_cast(input->dims()[1]); - const int img_h = static_cast(input->dims()[2]); - const int img_w = static_cast(input->dims()[3]); - const int f = static_cast(filter.dims()[0]); + int batch_size = static_cast(input->dims()[0]); + int img_c = static_cast(input->dims()[1]); + int img_h = static_cast(input->dims()[2]); + int img_w = static_cast(input->dims()[3]); + int f = static_cast(filter.dims()[0]); + bool is_nchw = true; + if (data_format == "NHWC") { + img_c = static_cast(input->dims()[3]); + img_h = static_cast(input->dims()[1]); + img_w = static_cast(input->dims()[2]); + is_nchw = false; + } const XPUT *input_data = reinterpret_cast(input->data()); const XPUT *filter_data = reinterpret_cast(filter.data()); @@ -64,7 +72,7 @@ class GemmConvXPUKernel : public framework::OpKernel { int r = xpu::conv2d( dev_ctx.x_context(), input_data, filter_data, output_data, batch_size, img_c, img_h, img_w, f, ksize, strides, paddings, dilations, groups, - nullptr, nullptr, nullptr, true); + nullptr, nullptr, nullptr, is_nchw); PADDLE_ENFORCE_EQ( r, XPU_SUCCESS, platform::errors::External("XPU conv kernel return wrong value[%d %s]", @@ -99,9 +107,9 @@ class GemmConvGradXPUKernel : public framework::OpKernel { context.Attr("padding_algorithm"); PADDLE_ENFORCE_EQ( - data_format == "NHWC" || data_format == "NDHWC", false, + data_format == "NDHWC", false, platform::errors::InvalidArgument( - ("XPU do support data_format is NCHW in conv grad op."))); + ("XPU doesn't support data_format is NDHWC in conv grad op."))); framework::DDim in_data_dims = phi::slice_ddim(input->dims(), 2, input->dims().size()); @@ -111,11 +119,18 @@ class GemmConvGradXPUKernel : public framework::OpKernel { UpdatePaddingAndDilation(&paddings, &dilations, padding_algorithm, in_data_dims, strides, ksize); - const int batch_size = static_cast(input->dims()[0]); - const int img_c = static_cast(input->dims()[1]); - const int img_h = static_cast(input->dims()[2]); - const int img_w = static_cast(input->dims()[3]); - const int f = static_cast(filter.dims()[0]); + int batch_size = static_cast(input->dims()[0]); + int img_c = static_cast(input->dims()[1]); + int img_h = static_cast(input->dims()[2]); + int img_w = static_cast(input->dims()[3]); + int f = static_cast(filter.dims()[0]); + bool is_nchw = true; + if (data_format == "NHWC") { + img_c = static_cast(input->dims()[3]); + img_h = static_cast(input->dims()[1]); + img_w = static_cast(input->dims()[2]); + is_nchw = false; + } const XPUT *input_data = reinterpret_cast(input->data()); const XPUT *filter_data = reinterpret_cast(filter.data()); @@ -136,7 +151,7 @@ class GemmConvGradXPUKernel : public framework::OpKernel { dev_ctx.x_context(), input_data, filter_data, output_grad_data, input_grad_data, filter_grad_data, batch_size, img_c, img_h, img_w, f, ksize, strides, paddings, dilations, groups, nullptr, nullptr, nullptr, - nullptr, nullptr, true); + nullptr, nullptr, is_nchw); PADDLE_ENFORCE_EQ( r, XPU_SUCCESS, platform::errors::External("XPU conv kernel return wrong value[%d %s]", diff --git a/paddle/fluid/operators/dropout_op_mlu.cc b/paddle/fluid/operators/dropout_op_mlu.cc new file mode 100644 index 0000000000000..b88974a51ceff --- /dev/null +++ b/paddle/fluid/operators/dropout_op_mlu.cc @@ -0,0 +1,165 @@ +/* Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + +#include "paddle/fluid/framework/op_registry.h" +#include "paddle/fluid/operators/mlu/mlu_baseop.h" + +namespace paddle { +namespace operators { + +using Tensor = framework::Tensor; + +template +class DropoutMLUKernel : public framework::OpKernel { + public: + void Compute(const framework::ExecutionContext& ctx) const override { + auto* x = ctx.Input("X"); + auto* out = ctx.Output("Out"); + auto dropout_prob = ctx.Attr("dropout_prob"); + auto is_test = ctx.Attr("is_test"); + auto* seed_tensor = + ctx.HasInput("Seed") ? ctx.Input("Seed") : nullptr; + auto dropout_implementation = + ctx.Attr("dropout_implementation"); + + const bool is_upscale = (dropout_implementation == "upscale_in_train"); + + out->mutable_data(ctx.GetPlace()); + MLUCnnlTensorDesc x_desc(*x); + MLUCnnlTensorDesc out_desc(*out); + + if (!is_test) { + // exec dropout op for training only. + int seed_data = 0; + if (seed_tensor) { + if (platform::is_mlu_place(seed_tensor->place())) { + memory::Copy(platform::CPUPlace(), &seed_data, seed_tensor->place(), + seed_tensor->data(), sizeof(int)); + } else { + seed_data = *(seed_tensor->data()); + } + } else { + seed_data = ctx.Attr("fix_seed") ? ctx.Attr("seed") : 0; + } + + auto* mask = ctx.Output("Mask"); + mask->mutable_data(ctx.GetPlace()); + MLUCnnlTensorDesc mask_desc(*mask); + // Special case when dropout_prob is 1.0 + if (dropout_prob == 1.0f) { + auto value_t = static_cast(0.0f); + MLUCnnl::Fill(ctx, CNNL_POINTER_MODE_HOST, &value_t, out_desc.get(), + GetBasePtr(out)); + MLUCnnl::Fill(ctx, CNNL_POINTER_MODE_HOST, &value_t, mask_desc.get(), + GetBasePtr(mask)); + return; + } + + // create mlu random generator + const int device_id = ctx.GetPlace().GetDeviceId(); + auto mlu_gen_random = GetMLURandomGenerator(ctx, device_id, seed_data); + + const float prob = is_upscale ? dropout_prob : 0.0f; + MLUCnnl::FusedDropout( + ctx, mlu_gen_random->get(), x_desc.get(), GetBasePtr(x), prob, + GetBasePtr(&(mlu_gen_random->get_state())), mask_desc.get(), + GetBasePtr(mask), out_desc.get(), GetBasePtr(out)); + } else { + // exec dropout op for inference only. + if (is_upscale) { + framework::TensorCopy( + *x, ctx.GetPlace(), + ctx.template device_context(), out); + } else { + float scale = static_cast(1.0f - dropout_prob); + Tensor scale_tensor(x->dtype()); + scale_tensor.mutable_data({1}, ctx.GetPlace()); + MLUCnnlTensorDesc scale_desc(scale_tensor); + MLUCnnl::Fill(ctx, CNNL_POINTER_MODE_HOST, &scale, scale_desc.get(), + GetBasePtr(&scale_tensor)); + + auto data_type = ToCnnlDataType(); + MLUCnnlOpTensorDesc op_tensor_desc(CNNL_OP_TENSOR_MUL, data_type, + CNNL_NOT_PROPAGATE_NAN); + MLUCnnl::OpTensor(ctx, op_tensor_desc.get(), x_desc.get(), + GetBasePtr(x), scale_desc.get(), + GetBasePtr(&scale_tensor), out_desc.get(), + GetBasePtr(out), data_type); + } + } + } +}; + +template +class DropoutGradMLUKernel : public framework::OpKernel { + public: + void Compute(const framework::ExecutionContext& ctx) const override { + PADDLE_ENFORCE_EQ(!ctx.Attr("is_test"), true, + platform::errors::InvalidArgument( + "GradOp is only callable when is_test is false")); + auto* grad_x = ctx.Output(framework::GradVarName("X")); + auto* grad_out = ctx.Input(framework::GradVarName("Out")); + auto* mask = ctx.Input("Mask"); + auto dropout_prob = ctx.Attr("dropout_prob"); + auto dropout_impl = ctx.Attr("dropout_implementation"); + + grad_x->mutable_data(ctx.GetPlace()); + MLUCnnlTensorDesc grad_x_desc(*grad_x); + + if (dropout_prob == 1.) { + auto value_t = static_cast(0.0f); + MLUCnnl::Fill(ctx, CNNL_POINTER_MODE_HOST, &value_t, grad_x_desc.get(), + GetBasePtr(grad_x)); + return; + } + + // cast mask from uint8 to float32/float16 + Tensor cast_mask(grad_x->dtype()); + cast_mask.Resize(mask->dims()); + cast_mask.mutable_data(ctx.GetPlace()); + + MLUCnnlTensorDesc mask_desc(*mask); + MLUCnnlTensorDesc cast_mask_desc(cast_mask); + cnnlCastDataType_t cast_type = + GetCastDataType(framework::TransToProtoVarType(mask->dtype()), + framework::TransToProtoVarType(cast_mask.dtype())); + + MLUCnnl::Cast(ctx, cast_type, mask_desc.get(), GetBasePtr(mask), + cast_mask_desc.get(), GetBasePtr(&cast_mask)); + + const bool is_upscale = (dropout_impl == "upscale_in_train"); + const float scale = is_upscale ? (1.0f / (1.0f - dropout_prob)) : (1.0f); + + auto data_type = ToCnnlDataType(); + MLUCnnlTensorDesc grad_out_desc(*grad_out); + MLUCnnlOpTensorDesc op_tensor_desc(CNNL_OP_TENSOR_MUL, data_type, + CNNL_NOT_PROPAGATE_NAN); + MLUCnnl::OpTensor(ctx, op_tensor_desc.get(), cast_mask_desc.get(), + GetBasePtr(&cast_mask), grad_out_desc.get(), + GetBasePtr(grad_out), grad_x_desc.get(), + GetBasePtr(grad_x), data_type, scale); + } +}; + +} // namespace operators +} // namespace paddle + +namespace ops = paddle::operators; +namespace plat = paddle::platform; + +REGISTER_OP_MLU_KERNEL(dropout, ops::DropoutMLUKernel, + ops::DropoutMLUKernel); + +REGISTER_OP_MLU_KERNEL(dropout_grad, ops::DropoutGradMLUKernel, + ops::DropoutGradMLUKernel); diff --git a/paddle/fluid/operators/einsum_op.cc b/paddle/fluid/operators/einsum_op.cc new file mode 100644 index 0000000000000..8cd8d94d6b389 --- /dev/null +++ b/paddle/fluid/operators/einsum_op.cc @@ -0,0 +1,95 @@ +// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include +#include +#include "paddle/fluid/framework/infershape_utils.h" +#include "paddle/fluid/framework/op_registry.h" +#include "paddle/fluid/framework/operator.h" +#include "paddle/phi/core/ddim.h" +#include "paddle/phi/kernels/impl/einsum_impl.h" + +namespace paddle { +namespace operators { +class EinsumOp : public framework::OperatorWithKernel { + public: + using framework::OperatorWithKernel::OperatorWithKernel; +}; + +class EinsumOpMaker : public framework::OpProtoAndCheckerMaker { + public: + void Make() override { + AddInput("Operands", "(TensorList), The input tensor of einsum op.") + .AsDuplicable(); + AddOutput("Out", "(Tensor), The output tensor of einsum op."); + AddAttr("equation", + "(string) A einsum equation. such as `ij,jk->ik`" + "There must have `->` and the number of operands in " + "equation must equals the `Operands` length."); + AddComment(R"DOC( +Einsum Operator. + +This operator is used to perform einsum operation for given operands and equation. +)DOC"); + } +}; + +class EinsumGradOp : public framework::OperatorWithKernel { + public: + using framework::OperatorWithKernel::OperatorWithKernel; + void InferShape(framework::InferShapeContext* ctx) const override { + auto x_name = "Operands"; + auto x_grad_name = framework::GradVarName(x_name); + ctx->SetOutputsDim(x_grad_name, ctx->GetInputsDim(x_name)); + ctx->ShareAllLoD(x_name, x_grad_name); + } + + protected: + framework::OpKernelType GetExpectedKernelType( + const framework::ExecutionContext& ctx) const override { + auto dtype = OperatorWithKernel::IndicateVarDataType( + ctx, framework::GradVarName("Out")); + return framework::OpKernelType(dtype, ctx.GetPlace()); + } +}; + +template +class EinsumGradMaker : public framework::SingleGradOpMaker { + public: + using framework::SingleGradOpMaker::SingleGradOpMaker; + + void Apply(GradOpPtr retv) const override { + retv->SetType("einsum_grad"); + retv->SetInput("Operands", this->Input("Operands")); + retv->SetInput(framework::GradVarName("Out"), this->OutputGrad("Out")); + retv->SetAttrMap(this->Attrs()); + retv->SetOutput(framework::GradVarName("Operands"), + this->InputGrad("Operands", false)); + } +}; + +} // namespace operators +} // namespace paddle + +namespace ops = paddle::operators; + +DECLARE_INFER_SHAPE_FUNCTOR(einsum, EinsumInferShapeFunctor, + PD_INFER_META(phi::EinsumInferShape)); + +REGISTER_OPERATOR(einsum, ops::EinsumOp, ops::EinsumOpMaker, + EinsumInferShapeFunctor, + ops::EinsumGradMaker, + ops::EinsumGradMaker); + +REGISTER_OPERATOR(einsum_grad, ops::EinsumGradOp); diff --git a/paddle/fluid/operators/fused/CMakeLists.txt b/paddle/fluid/operators/fused/CMakeLists.txt index 80e7f5c001d4b..68b9051d85831 100644 --- a/paddle/fluid/operators/fused/CMakeLists.txt +++ b/paddle/fluid/operators/fused/CMakeLists.txt @@ -19,6 +19,7 @@ register_operators(EXCLUDES fused_attention_op fused_transformer_op fused_feedforward_op + fused_multi_transformer_op resnet_unit_op fused_gemm_epilogue_op) @@ -73,6 +74,7 @@ if (WITH_GPU OR WITH_ROCM) op_library(fused_feedforward_op) # fused_attention_op op_library(fused_attention_op) + op_library(fused_multi_transformer_op) endif() # resnet_unit needs cudnn 8.0 above if ((NOT WITH_ROCM) AND (NOT ${CUDNN_VERSION} VERSION_LESS 8000)) diff --git a/paddle/fluid/operators/fused/fused_dropout_common.h b/paddle/fluid/operators/fused/fused_dropout_common.h index 6bf3a7114f4ce..0fe76fa23a637 100644 --- a/paddle/fluid/operators/fused/fused_dropout_common.h +++ b/paddle/fluid/operators/fused/fused_dropout_common.h @@ -43,9 +43,17 @@ inline platform::GpuLaunchConfig Get1DBlocksAnd2DGrids( const platform::CUDADeviceContext &ctx, const uint32_t rows, const uint32_t cols, const int vec_size) { const uint32_t tmp_cols = cols / vec_size; - int threads = std::max( - static_cast(32), - std::min(tmp_cols, static_cast(ctx.GetMaxThreadsPerBlock()))); + // NOTE(wangxi): We set max_block_size to 512, for `FusedResidualDropoutBias` + // needs too many register resources. If data_type is float16, CUDA + // error(701) will occur when block_size is 1024. Which error is + // 'cudaErrorLaunchOutOfResources', this indicates that a launch did not + // occur because it did not have appropriate resources. + // Of course, this kernel can be optimized later to reduce the use + // of registers. + int threads = + std::max(static_cast(32), + std::min(tmp_cols, static_cast(std::min( + ctx.GetMaxThreadsPerBlock(), 512)))); const auto blocks_x = std::max(static_cast(1), (tmp_cols + threads - 1) / threads); const auto blocks_y = std::max(static_cast(1), rows); diff --git a/paddle/fluid/operators/fused/fused_gemm_epilogue_op.cu b/paddle/fluid/operators/fused/fused_gemm_epilogue_op.cu index e16c9e8f483cc..9bf3d1a485efc 100644 --- a/paddle/fluid/operators/fused/fused_gemm_epilogue_op.cu +++ b/paddle/fluid/operators/fused/fused_gemm_epilogue_op.cu @@ -15,6 +15,7 @@ limitations under the License. */ #include "paddle/fluid/framework/op_registry.h" #include "paddle/fluid/framework/op_version_registry.h" +#include "paddle/fluid/operators/fused/fused_gemm_epilogue_op.h" #include "paddle/fluid/platform/dynload/cublasLt.h" #include "paddle/fluid/platform/float16.h" @@ -56,7 +57,6 @@ class FusedGemmEpilogueKernel : public framework::OpKernel { cublasComputeType_t compute_type = CUBLAS_COMPUTE_32F; if (std::is_same::value) { mat_type = CUDA_R_16F; - scale_type = CUDA_R_16F; } if (std::is_same::value) { mat_type = CUDA_R_64F; @@ -130,7 +130,7 @@ class FusedGemmEpilogueKernel : public framework::OpKernel { cublasLtHandle_t lt_handle = dev_ctx.cublaslt_handle(); size_t workspace_size = 4 * 1024 * 1024; - const cublasLtMatmulAlgo_t* algo = nullptr; + cudaStream_t stream = dev_ctx.stream(); memory::allocation::AllocationPtr workspace = memory::Alloc(dev_ctx, workspace_size); @@ -146,10 +146,26 @@ class FusedGemmEpilogueKernel : public framework::OpKernel { beta = &beta32; } + const auto* y_data = y->data(); + const auto* x_data = x->data(); + + cublasLtMatmulAlgo_t algo = GemmEpilogueAlgoCache::Instance().GetGemmAlgo( + lt_handle, operation_desc, y_desc, x_desc, out_desc, alpha, beta, + y_data, x_data, out_data, stream, workspace->ptr(), workspace_size); + PADDLE_ENFORCE_GPU_SUCCESS(platform::dynload::cublasLtMatmul( - lt_handle, operation_desc, alpha, y->data(), y_desc, x->data(), - x_desc, beta, out_data, out_desc, out_data, out_desc, algo, - workspace->ptr(), workspace_size, stream)); + lt_handle, operation_desc, alpha, y_data, y_desc, x_data, x_desc, beta, + out_data, out_desc, out_data, out_desc, &algo, workspace->ptr(), + workspace_size, stream)); + + PADDLE_ENFORCE_GPU_SUCCESS( + platform::dynload::cublasLtMatmulDescDestroy(operation_desc)); + PADDLE_ENFORCE_GPU_SUCCESS( + platform::dynload::cublasLtMatrixLayoutDestroy(y_desc)); + PADDLE_ENFORCE_GPU_SUCCESS( + platform::dynload::cublasLtMatrixLayoutDestroy(x_desc)); + PADDLE_ENFORCE_GPU_SUCCESS( + platform::dynload::cublasLtMatrixLayoutDestroy(out_desc)); } private: @@ -205,7 +221,6 @@ class FusedGemmEpilogueGradKernel : public framework::OpKernel { cublasComputeType_t compute_type = CUBLAS_COMPUTE_32F; if (std::is_same::value) { mat_type = CUDA_R_16F; - scale_type = CUDA_R_16F; } if (std::is_same::value) { mat_type = CUDA_R_64F; @@ -215,7 +230,6 @@ class FusedGemmEpilogueGradKernel : public framework::OpKernel { cublasLtHandle_t lt_handle = dev_ctx.cublaslt_handle(); size_t workspace_size = 4 * 1024 * 1024; - const cublasLtMatmulAlgo_t* algo = nullptr; cudaStream_t stream = dev_ctx.stream(); double alpha64 = 1.0, beta64 = 0.0; @@ -262,8 +276,8 @@ class FusedGemmEpilogueGradKernel : public framework::OpKernel { &aux_data, sizeof(aux_data))); PADDLE_ENFORCE_GPU_SUCCESS( platform::dynload::cublasLtMatmulDescSetAttribute( - dx_operation_desc, CUBLASLT_MATMUL_DESC_EPILOGUE_AUX_LD, &N, - sizeof(N))); + dx_operation_desc, CUBLASLT_MATMUL_DESC_EPILOGUE_AUX_LD, &K, + sizeof(K))); } cublasLtMatrixLayout_t y_desc = NULL, dx_desc = NULL; @@ -277,10 +291,24 @@ class FusedGemmEpilogueGradKernel : public framework::OpKernel { dx->mutable_data(ctx.GetPlace()); auto* dx_data = dx->data(); + const auto* y_data = y->data(); + const auto* dout_data = dout->data(); + + cublasLtMatmulAlgo_t algo = GemmEpilogueAlgoCache::Instance().GetGemmAlgo( + lt_handle, dx_operation_desc, y_desc, dout_desc, dx_desc, alpha, beta, + y_data, dout_data, dx_data, stream, dx_workspace->ptr(), + workspace_size); + PADDLE_ENFORCE_GPU_SUCCESS(platform::dynload::cublasLtMatmul( lt_handle, dx_operation_desc, alpha, y->data(), y_desc, dout->data(), dout_desc, beta, dx_data, dx_desc, dx_data, dx_desc, - algo, dx_workspace->ptr(), workspace_size, stream)); + &algo, dx_workspace->ptr(), workspace_size, stream)); + PADDLE_ENFORCE_GPU_SUCCESS( + platform::dynload::cublasLtMatmulDescDestroy(dx_operation_desc)); + PADDLE_ENFORCE_GPU_SUCCESS( + platform::dynload::cublasLtMatrixLayoutDestroy(y_desc)); + PADDLE_ENFORCE_GPU_SUCCESS( + platform::dynload::cublasLtMatrixLayoutDestroy(dx_desc)); } if (dy) { @@ -324,11 +352,27 @@ class FusedGemmEpilogueGradKernel : public framework::OpKernel { dy->mutable_data(ctx.GetPlace()); auto* dy_data = dy->data(); + const auto* dout_data = dout->data(); + const auto* x_data = x->data(); + + cublasLtMatmulAlgo_t algo = GemmEpilogueAlgoCache::Instance().GetGemmAlgo( + lt_handle, dy_operation_desc, dout_desc, x_desc, dy_desc, alpha, beta, + dout_data, x_data, dy_data, stream, dy_workspace->ptr(), + workspace_size); + PADDLE_ENFORCE_GPU_SUCCESS(platform::dynload::cublasLtMatmul( - lt_handle, dy_operation_desc, alpha, dout->data(), dout_desc, - x->data(), x_desc, beta, dy_data, dy_desc, dy_data, dy_desc, algo, + lt_handle, dy_operation_desc, alpha, dout_data, dout_desc, x_data, + x_desc, beta, dy_data, dy_desc, dy_data, dy_desc, &algo, dy_workspace->ptr(), workspace_size, stream)); + PADDLE_ENFORCE_GPU_SUCCESS( + platform::dynload::cublasLtMatmulDescDestroy(dy_operation_desc)); + PADDLE_ENFORCE_GPU_SUCCESS( + platform::dynload::cublasLtMatrixLayoutDestroy(x_desc)); + PADDLE_ENFORCE_GPU_SUCCESS( + platform::dynload::cublasLtMatrixLayoutDestroy(dy_desc)); } + PADDLE_ENFORCE_GPU_SUCCESS( + platform::dynload::cublasLtMatrixLayoutDestroy(dout_desc)); } private: diff --git a/paddle/fluid/operators/fused/fused_gemm_epilogue_op.h b/paddle/fluid/operators/fused/fused_gemm_epilogue_op.h new file mode 100644 index 0000000000000..c90a6966fe0a8 --- /dev/null +++ b/paddle/fluid/operators/fused/fused_gemm_epilogue_op.h @@ -0,0 +1,271 @@ +/* Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +Copyright (c) 2022 NVIDIA Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + +#pragma once + +#include +#include +#include +#include +#include "gflags/gflags.h" +#include "paddle/fluid/platform/dynload/cublasLt.h" +#include "paddle/fluid/platform/float16.h" + +DECLARE_int64(cublaslt_exhaustive_search_times); + +namespace paddle { +namespace operators { + +class GemmEpilogueAlgoCache { + public: + static GemmEpilogueAlgoCache &Instance() { + static GemmEpilogueAlgoCache instance( + FLAGS_cublaslt_exhaustive_search_times); + return instance; + } + + GemmEpilogueAlgoCache(GemmEpilogueAlgoCache const &) = delete; + void operator=(GemmEpilogueAlgoCache const &) = delete; + + cublasLtMatmulAlgo_t GetGemmAlgo( + cublasLtHandle_t lt_handle, cublasLtMatmulDesc_t op_desc, + cublasLtMatrixLayout_t a_desc, cublasLtMatrixLayout_t b_desc, + cublasLtMatrixLayout_t c_desc, const void *alpha, const void *beta, + const void *a, const void *b, void *c, cudaStream_t stream, + void *workspace, size_t workspace_size) { + int64_t seed = 0; + std::hash hash_fn; + + HashMatmulDesc_(op_desc, &seed, hash_fn); + HashMatrixLayoutDesc_(a_desc, &seed, hash_fn); + HashMatrixLayoutDesc_(b_desc, &seed, hash_fn); + HashMatrixLayoutDesc_(c_desc, &seed, hash_fn); + + cublasLtMatmulAlgo_t ret; + auto it = map_.end(); + bool have_found = false; + { + std::lock_guard lock(cache_mutex_); + it = map_.find(seed); + + if (it != map_.end()) { + ret = it->second; + have_found = true; + } + } + + if (!have_found) { + cublasLtMatmulPreference_t preference; + PADDLE_ENFORCE_GPU_SUCCESS( + platform::dynload::cublasLtMatmulPreferenceCreate(&preference)); + PADDLE_ENFORCE_GPU_SUCCESS( + platform::dynload::cublasLtMatmulPreferenceSetAttribute( + preference, CUBLASLT_MATMUL_PREF_MAX_WORKSPACE_BYTES, + &workspace_size, sizeof(workspace_size))); + + int returned_results = 0; + cublasLtMatmulHeuristicResult_t heuristic_results[requested_algo_count_] = + {0}; + PADDLE_ENFORCE_GPU_SUCCESS( + platform::dynload::cublasLtMatmulAlgoGetHeuristic( + lt_handle, op_desc, a_desc, b_desc, c_desc, c_desc, preference, + requested_algo_count_, heuristic_results, &returned_results)); + + PADDLE_ENFORCE_GT( + returned_results, 0, + platform::errors::Unavailable("No GEMM epilogue algorithm support!")); + + PADDLE_ENFORCE_GPU_SUCCESS( + platform::dynload::cublasLtMatmulPreferenceDestroy(preference)); + + if (search_times_ > 0) { + int best_algo_idx = -1; + float best_algo_time = 0; + + // Run 100 times for warmup + int warmup_algo_idx = 0; + for (int t = 0; t < 100; t++) { + cublasStatus_t status = platform::dynload::cublasLtMatmul( + lt_handle, op_desc, alpha, a, a_desc, b, b_desc, beta, c, c_desc, + c, c_desc, &heuristic_results[warmup_algo_idx].algo, workspace, + workspace_size, stream); + if (status != CUBLAS_STATUS_SUCCESS) { + t = -1; + warmup_algo_idx += 1; + if (warmup_algo_idx == requested_algo_count_) { + PADDLE_THROW(platform::errors::Unavailable( + "No GEMM epilogue algorithm support!")); + } + } + } + + cudaEvent_t start_event, stop_event; + PADDLE_ENFORCE_GPU_SUCCESS(cudaEventCreate(&start_event)); + PADDLE_ENFORCE_GPU_SUCCESS(cudaEventCreate(&stop_event)); + + for (int algo_idx = 0; algo_idx < returned_results; ++algo_idx) { + float curr_time = 0; + for (int check_idx = 0; check_idx < search_times_; check_idx++) { + float time = 0; + PADDLE_ENFORCE_GPU_SUCCESS(cudaEventRecord(start_event, stream)); + + cublasStatus_t status = platform::dynload::cublasLtMatmul( + lt_handle, op_desc, alpha, a, a_desc, b, b_desc, beta, c, + c_desc, c, c_desc, &heuristic_results[algo_idx].algo, workspace, + workspace_size, stream); + + PADDLE_ENFORCE_GPU_SUCCESS(cudaEventRecord(stop_event, stream)); + PADDLE_ENFORCE_GPU_SUCCESS(cudaEventSynchronize(stop_event)); + PADDLE_ENFORCE_GPU_SUCCESS( + cudaEventElapsedTime(&time, start_event, stop_event)); + curr_time += time; + if (status != CUBLAS_STATUS_SUCCESS) { + curr_time = 3.40282e+038; // Max Value of float + break; + } + } + + curr_time = curr_time / search_times_; + if (curr_time < best_algo_time || algo_idx == 0) { + best_algo_idx = algo_idx; + best_algo_time = curr_time; + } + } + + PADDLE_ENFORCE_GPU_SUCCESS(cudaEventDestroy(start_event)); + PADDLE_ENFORCE_GPU_SUCCESS(cudaEventDestroy(stop_event)); + + if (best_algo_idx == -1) { + PADDLE_THROW(platform::errors::Unavailable( + "No GEMM epilogue algorithm support!")); + } + + ret = heuristic_results[best_algo_idx].algo; + } else { + int decided_algo_idx = -1; + for (int algo_idx = 0; algo_idx < returned_results; ++algo_idx) { + cublasStatus_t status = platform::dynload::cublasLtMatmul( + lt_handle, op_desc, alpha, a, a_desc, b, b_desc, beta, c, c_desc, + c, c_desc, &heuristic_results[algo_idx].algo, workspace, + workspace_size, stream); + if (status == CUBLAS_STATUS_SUCCESS) { + decided_algo_idx = algo_idx; + break; + } + } + if (decided_algo_idx == -1) { + PADDLE_THROW(platform::errors::Unavailable( + "No GEMM epilogue algorithm support!")); + } + ret = heuristic_results[decided_algo_idx].algo; + } + + std::lock_guard lock(cache_mutex_); + map_[seed] = ret; + } + + VLOG(4) << "Search time:" << search_times_ << ", Is hash-key (" << seed + << ") found in GemmEpilogueAlgoCache? " << have_found; + + return ret; + } + + private: + explicit GemmEpilogueAlgoCache(int search_times) + : search_times_(search_times) { + map_.clear(); + } + std::unordered_map map_; + int search_times_; + const int requested_algo_count_ = 10; + std::mutex cache_mutex_; + + void HashMatmulDesc_(cublasLtMatmulDesc_t desc, int64_t *seed, + const std::hash &hash_fn) { + size_t size_to_write; + int trans_a, trans_b; + uint32_t epilogue; + + PADDLE_ENFORCE_GPU_SUCCESS( + platform::dynload::cublasLtMatmulDescGetAttribute( + desc, CUBLASLT_MATMUL_DESC_TRANSA, &trans_a, sizeof(trans_a), + &size_to_write)); + HashValue_(seed, hash_fn, static_cast(trans_a)); + + PADDLE_ENFORCE_GPU_SUCCESS( + platform::dynload::cublasLtMatmulDescGetAttribute( + desc, CUBLASLT_MATMUL_DESC_TRANSB, &trans_b, sizeof(trans_b), + &size_to_write)); + HashValue_(seed, hash_fn, static_cast(trans_b)); + + PADDLE_ENFORCE_GPU_SUCCESS( + platform::dynload::cublasLtMatmulDescGetAttribute( + desc, CUBLASLT_MATMUL_DESC_EPILOGUE, &epilogue, sizeof(epilogue), + &size_to_write)); + HashValue_(seed, hash_fn, static_cast(epilogue)); + } + + void HashMatrixLayoutDesc_(cublasLtMatrixLayout_t desc, int64_t *seed, + const std::hash &hash_fn) { + size_t size_to_write; + uint32_t dtype; + int32_t batch; + uint64_t row, col; + int64_t ld, batch_offset; + + PADDLE_ENFORCE_GPU_SUCCESS( + platform::dynload::cublasLtMatrixLayoutGetAttribute( + desc, CUBLASLT_MATRIX_LAYOUT_TYPE, &dtype, sizeof(dtype), + &size_to_write)); + HashValue_(seed, hash_fn, static_cast(dtype)); + + PADDLE_ENFORCE_GPU_SUCCESS( + platform::dynload::cublasLtMatrixLayoutGetAttribute( + desc, CUBLASLT_MATRIX_LAYOUT_BATCH_COUNT, &batch, sizeof(batch), + &size_to_write)); + HashValue_(seed, hash_fn, static_cast(batch)); + + PADDLE_ENFORCE_GPU_SUCCESS( + platform::dynload::cublasLtMatrixLayoutGetAttribute( + desc, CUBLASLT_MATRIX_LAYOUT_ROWS, &row, sizeof(row), + &size_to_write)); + HashValue_(seed, hash_fn, static_cast(row)); + + PADDLE_ENFORCE_GPU_SUCCESS( + platform::dynload::cublasLtMatrixLayoutGetAttribute( + desc, CUBLASLT_MATRIX_LAYOUT_COLS, &col, sizeof(col), + &size_to_write)); + HashValue_(seed, hash_fn, static_cast(col)); + + PADDLE_ENFORCE_GPU_SUCCESS( + platform::dynload::cublasLtMatrixLayoutGetAttribute( + desc, CUBLASLT_MATRIX_LAYOUT_LD, &ld, sizeof(ld), &size_to_write)); + HashValue_(seed, hash_fn, static_cast(ld)); + + PADDLE_ENFORCE_GPU_SUCCESS( + platform::dynload::cublasLtMatrixLayoutGetAttribute( + desc, CUBLASLT_MATRIX_LAYOUT_STRIDED_BATCH_OFFSET, &batch_offset, + sizeof(batch_offset), &size_to_write)); + HashValue_(seed, hash_fn, static_cast(batch_offset)); + } + + void HashValue_(int64_t *seed, const std::hash &hash_fn, + int64_t value) { + *seed ^= hash_fn(value) + 0x9e3779b9 + (*seed << 6) + (*seed >> 2); + } +}; + +} // namespace operators +} // namespace paddle diff --git a/paddle/fluid/operators/fused/fused_layernorm_residual_dropout_bias.h b/paddle/fluid/operators/fused/fused_layernorm_residual_dropout_bias.h index d53a24a57e3cc..aa613dd3f5ce0 100644 --- a/paddle/fluid/operators/fused/fused_layernorm_residual_dropout_bias.h +++ b/paddle/fluid/operators/fused/fused_layernorm_residual_dropout_bias.h @@ -156,9 +156,9 @@ __global__ void FusedLayernormResidualDropoutBias( } /* -* @brief layernorm(residual + dropout(x)); + * @brief layernorm(residual + dropout(x)); * Conditions: - * (1) The number of cols is 1024; + * (1) The number of cols is 768/1024/4096; * (2) layer_norm scale and bias is not null; * (3) linear bias is null; * @param @@ -166,6 +166,7 @@ __global__ void FusedLayernormResidualDropoutBias( * cols: 1024 * x_: [rows, cols], inputs * residual_:[rows, cols] + * bias_: [cols], linear bias, can be null * gamma_: [cols]: layernorm scale, not null * beta_: [cols], layernorm bias, not null * mask_out_: [rows, cols], dropout result @@ -173,7 +174,7 @@ __global__ void FusedLayernormResidualDropoutBias( * y_: [rows, cols], layernorm result * mean_out_: [rows]: layernorm means * var_out_: [rows]: layernorm vars -*/ + */ template < typename T, typename U, typename ScaleT = U, typename MaskType = uint8_t, int VecSize = 8, int WARPS_M = 4, int WARPS_N = 1, int BYTES_PER_LDG = 16, @@ -182,14 +183,16 @@ template < int THREADS_PER_CTA = WARPS_M *THREADS_PER_ROW, int ROWS_PER_CTA = WARPS_M, int ELTS_PER_ROW_PER_CTA = THREADS_PER_ROW *VecSize, int LDGS = ELTS_PER_ROW / ELTS_PER_ROW_PER_CTA> -__global__ __launch_bounds__(THREADS_PER_CTA) void fused_ln_fwd_1024_kernel( +__global__ __launch_bounds__(THREADS_PER_CTA) void fused_fast_ln_fwd_kernel( int rows, int cols, uint64_t seed, const float dropout_prob, const bool is_upscale_in_train, const bool is_test, const uint64_t increment, const float epsilon, const T *__restrict__ x_ptr, - const T *__restrict__ residual_ptr, const ScaleT *__restrict__ gamma_ptr, - const ScaleT *__restrict__ beta_ptr, MaskType *__restrict__ mask_out_ptr, - U *__restrict__ mean_out_ptr, U *__restrict__ var_out_ptr, - T *__restrict__ residual_out_ptr, T *__restrict__ y_ptr) { + const T *__restrict__ residual_ptr, const T *__restrict__ bias_ptr, + const ScaleT *__restrict__ gamma_ptr, const ScaleT *__restrict__ beta_ptr, + MaskType *__restrict__ mask_out_ptr, U *__restrict__ mean_out_ptr, + U *__restrict__ var_out_ptr, T *__restrict__ residual_out_ptr, + T *__restrict__ y_ptr) { + __shared__ U smem[WARPS_M * WARPS_N]; using Vec = phi::AlignedVector; using Vec_scale = phi::AlignedVector; using MaskStoreT = phi::AlignedVector; @@ -204,12 +207,22 @@ __global__ __launch_bounds__(THREADS_PER_CTA) void fused_ln_fwd_1024_kernel( const int c = warp_n * THREADS_PER_WARP + lane; // lane const int r = bidx * ROWS_PER_CTA + warp_m; // row id - int idx = r * LN_NUM_COLS + c; + int idx = r * ELTS_PER_ROW + c; curandStatePhilox4_32_10_t state; curand_init(seed, idx, increment, &state); T factor = GetFactor(dropout_prob, is_upscale_in_train, is_test); + // bias + Vec bias[LDGS]; + if (bias_ptr != nullptr) { +#pragma unroll + for (int it = 0, col = c; it < LDGS; it++) { + phi::Load(bias_ptr + col * VecSize, &bias[it]); + col += THREADS_PER_ROW; + } + } + Vec_scale gamma[LDGS]; Vec_scale beta[LDGS]; #pragma unroll @@ -219,14 +232,14 @@ __global__ __launch_bounds__(THREADS_PER_CTA) void fused_ln_fwd_1024_kernel( col += THREADS_PER_ROW; } - constexpr U rn = 1.f / U(LN_NUM_COLS); + constexpr U rn = 1.f / U(ELTS_PER_ROW); for (int row = r; row < rows; row += gridDim.x * ROWS_PER_CTA) { Vec x[LDGS]; Vec residual[LDGS]; #pragma unroll for (int it = 0, col = c; it < LDGS; it++) { - phi::Load(x_ptr + row * LN_NUM_COLS + col * VecSize, &x[it]); - phi::Load(residual_ptr + row * LN_NUM_COLS + col * VecSize, + phi::Load(x_ptr + row * ELTS_PER_ROW + col * VecSize, &x[it]); + phi::Load(residual_ptr + row * ELTS_PER_ROW + col * VecSize, &residual[it]); col += THREADS_PER_ROW; } @@ -255,14 +268,28 @@ __global__ __launch_bounds__(THREADS_PER_CTA) void fused_ln_fwd_1024_kernel( // 4 * 8 U xf[LDGS * VecSize]; + if (bias_ptr != nullptr) { #pragma unroll - for (int it = 0; it < LDGS; it++) { + for (int it = 0; it < LDGS; it++) { #pragma unroll - for (int jt = 0; jt < VecSize; jt++) { - // dropout(x) + residual - x[it][jt] = x[it][jt] * static_cast(mask_vec[it][jt]) * factor + - residual[it][jt]; - xf[it * VecSize + jt] = U(x[it][jt]); + for (int jt = 0; jt < VecSize; jt++) { + // dropout(x) + residual + x[it][jt] = (x[it][jt] + bias[it][jt]) * + static_cast(mask_vec[it][jt]) * factor + + residual[it][jt]; + xf[it * VecSize + jt] = U(x[it][jt]); + } + } + } else { +#pragma unroll + for (int it = 0; it < LDGS; it++) { +#pragma unroll + for (int jt = 0; jt < VecSize; jt++) { + // dropout(x) + residual + x[it][jt] = x[it][jt] * static_cast(mask_vec[it][jt]) * factor + + residual[it][jt]; + xf[it * VecSize + jt] = U(x[it][jt]); + } } } @@ -270,9 +297,9 @@ __global__ __launch_bounds__(THREADS_PER_CTA) void fused_ln_fwd_1024_kernel( #pragma unroll for (int it = 0, col = c; it < LDGS; it++) { phi::Store( - x[it], residual_out_ptr + row * LN_NUM_COLS + col * VecSize); + x[it], residual_out_ptr + row * ELTS_PER_ROW + col * VecSize); phi::Store( - mask_vec[it], mask_out_ptr + row * LN_NUM_COLS + col * VecSize); + mask_vec[it], mask_out_ptr + row * ELTS_PER_ROW + col * VecSize); col += THREADS_PER_ROW; } @@ -289,6 +316,22 @@ __global__ __launch_bounds__(THREADS_PER_CTA) void fused_ln_fwd_1024_kernel( for (int it = 1; it < THREADS_PER_WARP; it *= 2) { mu_local += __shfl_xor_sync(uint32_t(-1), mu_local, it); } + if (WARPS_N > 1) { + if (lane == 0) { + smem[warp_m * WARPS_N + warp_n] = mu_local; + } + __syncthreads(); + if (tidx == 0) { + mu_local = 0.f; +#pragma unroll + for (int it = 0; it < WARPS_N; ++it) { + mu_local += smem[warp_m * WARPS_N + it]; + } + smem[warp_m] = mu_local; + } + __syncthreads(); + mu_local = smem[warp_m]; + } mu_local *= rn; if (lane == 0) { mean_out_ptr[row] = mu_local; @@ -308,6 +351,22 @@ __global__ __launch_bounds__(THREADS_PER_CTA) void fused_ln_fwd_1024_kernel( for (int it = 1; it < THREADS_PER_WARP; it *= 2) { var_local += __shfl_xor_sync(uint32_t(-1), var_local, it); } + if (WARPS_N > 1) { + if (lane == 0) { + smem[warp_m * WARPS_N + warp_n] = var_local; + } + __syncthreads(); + if (tidx == 0) { + var_local = 0.f; +#pragma unroll + for (int it = 0; it < WARPS_N; ++it) { + var_local += smem[warp_m * WARPS_N + it]; + } + smem[warp_m] = var_local; + } + __syncthreads(); + var_local = smem[warp_m]; + } U rsigma = rsqrtf(var_local * rn + epsilon); if (lane == 0) { // Note: the stored var is different for paddle(ln) and apex (fast ln). @@ -332,7 +391,7 @@ __global__ __launch_bounds__(THREADS_PER_CTA) void fused_ln_fwd_1024_kernel( #pragma unroll for (int it = 0, col = c; it < LDGS; it++) { - phi::Store(x[it], y_ptr + row * LN_NUM_COLS + col * VecSize); + phi::Store(x[it], y_ptr + row * ELTS_PER_ROW + col * VecSize); col += THREADS_PER_ROW; } } @@ -390,12 +449,37 @@ void LaunchLayernormResidualDropoutBias( return; } - bool can_call_1024_kernel = false; - if (cols == 1024 && scale != nullptr && layernorm_bias != nullptr && - bias == nullptr) { - can_call_1024_kernel = true; +#define LAUNCH_FUSED_FAST_LN_KERNEL_BASE(cols) \ + case (cols): { \ + constexpr int WARPS_N = cols < 1024 ? 1 : (cols / 1024); \ + constexpr int WARPS_M = 4 / WARPS_N; \ + const int THREADS_PER_WARP = 32; \ + const int BYTES_PER_LDG = 16; \ + const int VecSize = BYTES_PER_LDG / sizeof(T); \ + const int THREADS_PER_CTA = WARPS_N * THREADS_PER_WARP * WARPS_M; \ + const int ROWS_PER_CTA = WARPS_M; \ + const int grid = \ + static_cast(std::ceil(rows / static_cast(ROWS_PER_CTA))); \ + fused_fast_ln_fwd_kernel< \ + T, U, LayerNormScaleBiasT, uint8_t, \ + VecSize, WARPS_M, WARPS_N, BYTES_PER_LDG, \ + cols><<>>( \ + rows, cols, seed, dropout_prob, is_upscale_in_train, is_test, \ + increment, epsilon, src, residual, bias, scale, layernorm_bias, \ + mask_data, mean, var, dst, layernorm_dst); \ + } break + +#define LAUNCH_FUSED_FAST_LN_KERNEL \ + LAUNCH_FUSED_FAST_LN_KERNEL_BASE(768); \ + LAUNCH_FUSED_FAST_LN_KERNEL_BASE(1024); \ + LAUNCH_FUSED_FAST_LN_KERNEL_BASE(4096) + + bool can_call_fast_ln_kernel = false; + if ((cols == 768 || cols == 1024 || cols == 4096) && scale != nullptr && + layernorm_bias != nullptr) { + can_call_fast_ln_kernel = true; } - VLOG(6) << "can_call_1024_kernel = " << can_call_1024_kernel; + VLOG(6) << "can_call_fast_ln_kernel = " << can_call_fast_ln_kernel; const int VecSize = MAX_CACHE_BYTES / sizeof(T); if (cols % VecSize != 0) { @@ -407,26 +491,15 @@ void LaunchLayernormResidualDropoutBias( epsilon, src, residual, bias, scale, layernorm_bias, mask_data, dst, layernorm_dst, mean, var); } else { - if (can_call_1024_kernel) { - const int WARPS_M = 4; - const int WARPS_N = 1; - const int THREADS_PER_WARP = 32; - const int BYTES_PER_LDG = 16; - const int VecSize = BYTES_PER_LDG / sizeof(T); - - const int THREADS_PER_CTA = WARPS_N * THREADS_PER_WARP * WARPS_M; - const int ROWS_PER_CTA = WARPS_M; - - // Note: the grid can not exceed max_grid of the gpu. - const int grid = - static_cast(std::ceil(rows / static_cast(ROWS_PER_CTA))); - fused_ln_fwd_1024_kernel< - T, U, LayerNormScaleBiasT, uint8_t, - VecSize, WARPS_M, WARPS_N, - BYTES_PER_LDG><<>>( - rows, cols, seed, dropout_prob, is_upscale_in_train, is_test, - increment, epsilon, src, residual, scale, layernorm_bias, mask_data, - mean, var, dst, layernorm_dst); + if (can_call_fast_ln_kernel) { + switch (cols) { + LAUNCH_FUSED_FAST_LN_KERNEL; + default: + PADDLE_THROW(platform::errors::InvalidArgument( + "Only when column is equal to 768/1024/4096 is supported for " + "now")); + break; + } } else { int blockDim = GetDesiredBlockDim(cols / VecSize); FusedLayernormResidualDropoutBias< diff --git a/paddle/fluid/operators/fused/fused_multi_transformer_op.cc b/paddle/fluid/operators/fused/fused_multi_transformer_op.cc new file mode 100644 index 0000000000000..c95ca6fe0c96c --- /dev/null +++ b/paddle/fluid/operators/fused/fused_multi_transformer_op.cc @@ -0,0 +1,259 @@ +/* Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + +#include +#include +#include "paddle/fluid/framework/op_registry.h" + +namespace paddle { +namespace operators { + +using Tensor = framework::Tensor; + +class FusedMultiTransformerOp : public framework::OperatorWithKernel { + private: + static constexpr const char *OpName = "FusedMultiTransformerOp"; + + public: + using framework::OperatorWithKernel::OperatorWithKernel; + + void InferShape(framework::InferShapeContext *ctx) const override { +#define CHECK_INPUT(name) \ + OP_INOUT_CHECK(ctx->HasInput(#name), "Input", #name, OpName) +#define CHECK_INPUTS(name) \ + OP_INOUT_CHECK(ctx->HasInputs(#name), "Input", #name, OpName) +#define CHECK_OUTPUT(name) \ + OP_INOUT_CHECK(ctx->HasOutput(#name), "Output", #name, OpName) +#define CHECK_OUTPUTS(name) \ + OP_INOUT_CHECK(ctx->HasOutputs(#name), "Output", #name, OpName) + + CHECK_INPUT(X); + + // attention + CHECK_INPUTS(QKVW); + CHECK_INPUTS(OutLinearW); + + if (ctx->HasInput("TimeStep")) { + CHECK_INPUTS(CacheKV); + } + + if (ctx->HasInputs("CacheKV")) { + CHECK_OUTPUTS(CacheKVOut); + } + + // ffn + CHECK_INPUTS(FFN1Weight); + CHECK_INPUTS(FFN2Weight); + + CHECK_OUTPUT(Out); + + // x: qkv's input [batch_size, seq_len, dim_embed] + // y: qkv's weight: [3, num_head, dim_head, dim_embed] + auto x_dim = ctx->GetInputDim("X"); + auto y_dim = ctx->GetInputsDim("QKVW")[0]; + PADDLE_ENFORCE_EQ(x_dim.size(), 3, platform::errors::InvalidArgument( + "The dimensions of x must be 3" + "(batch_size, seq_len, dim_embed)," + "but received dimensions of" + "Input is [%d]", + x_dim.size())); + PADDLE_ENFORCE_EQ(y_dim.size(), 4, + platform::errors::InvalidArgument( + "The dimensions of qkv_weight must be 4" + "(3, num_head, dim_head, dim_embed)," + "but received dimensions of" + "Input is [%d]", + y_dim.size())); + PADDLE_ENFORCE_EQ(x_dim[2], y_dim[3], + platform::errors::InvalidArgument( + "ShapeError: the dimension of x_dim[2] and y_dim[3]" + "must be equal. But received: the shape " + "of input x = [%s], and the shape of " + "input qkv_weight = [%s]", + x_dim, y_dim)); + + if (ctx->Attrs().Get("ring_id") == -1) { + PADDLE_ENFORCE_EQ(y_dim[1] * y_dim[2], y_dim[3], + platform::errors::InvalidArgument( + "The dimensions of qkv_weight must be 4" + "(3, num_head, dim_head, dim_embed)," + "and must satisfy the limitations: " + "(num_head * dim_head == dim_embed)")); + } + + if (ctx->HasInputs("CacheKV")) { + // [2, batch_size, num_head, max_seq_len, head_size] + const auto &c_dims = ctx->GetInputsDim("CacheKV"); + const auto &c_dim = c_dims[0]; + + PADDLE_ENFORCE_EQ( + c_dim.size(), 5, + paddle::platform::errors::InvalidArgument( + "The CacheKV must be 5 dims, but got %d", c_dim.size())); + PADDLE_ENFORCE_EQ(c_dim[0], 2, + paddle::platform::errors::InvalidArgument( + "The first dim of CacheKV must be 2, but got %d", + c_dim[0])); // 2 + PADDLE_ENFORCE_EQ(c_dim[1], x_dim[0], + paddle::platform::errors::InvalidArgument( + "The second dim of CacheKV must be equal with " + "batch size %d, but got %d", + x_dim[0], c_dim[1])); // batch_size + PADDLE_ENFORCE_EQ(c_dim[2], y_dim[1], + paddle::platform::errors::InvalidArgument( + "The third dim of CacheKV must be equal with num " + "head %d, but got %d", + y_dim[1], c_dim[2])); // num_head + PADDLE_ENFORCE_GT( + c_dim[3], 0, + paddle::platform::errors::InvalidArgument( + "The forth dim of CacheKV must be greater than 0, but got %d", + c_dim[3])); // cache_seq_len + PADDLE_ENFORCE_EQ(c_dim[4], y_dim[2], + paddle::platform::errors::InvalidArgument( + "The fifth dim of CacheKV must be equal with head " + "size %d, but got %d", + y_dim[2], c_dim[4])); // head_size + } + + ctx->SetOutputDim("Out", ctx->GetInputDim("X")); + } + + protected: + framework::OpKernelType GetExpectedKernelType( + const framework::ExecutionContext &ctx) const override { + return framework::OpKernelType( + OperatorWithKernel::IndicateVarDataType(ctx, "X"), ctx.GetPlace()); + } + + framework::OpKernelType GetKernelTypeForVar( + const std::string &var_name, const Tensor &tensor, + const framework::OpKernelType &expected_kernel_type) const override { + if (var_name == "TimeStep") { + VLOG(10) << "var_name:" << var_name << " need not to transform"; + return expected_kernel_type; + } + return framework::OpKernelType(expected_kernel_type.data_type_, + tensor.place(), tensor.layout()); + } +}; + +class FusedMultiTransformerOpOpMaker + : public framework::OpProtoAndCheckerMaker { + public: + void Make() override { + AddInput("X", "The input tensor."); + AddInput("LnScale", + "Scale is a 1-dimensional tensor of size " + "H. Here, H represents the last dimension of its input tensor.") + .AsDuplicable(); + AddInput("LnBias", + "Bias is a 1-dimensional tensor of size " + "H. Here, H represents the last dimension of its input tensor.") + .AsDuplicable(); + AddInput("QKVW", "The qkv weight tensor.").AsDuplicable(); + AddInput("QKVBias", "The qkv bias tensor.").AsDispensable().AsDuplicable(); + AddInput("CacheKV", "(optional) The cached KV for generation inference.") + .AsDispensable() + .AsDuplicable(); + AddInput("TimeStep", + "(optional, int) The time step for generation inference.") + .AsDispensable(); + AddInput("SrcMask", "(optional) The attention mask tensor in fmha.") + .AsDispensable(); + AddInput("OutLinearW", "The out_linear weight tensor.").AsDuplicable(); + AddInput("OutLinearBias", "The out_linear bias tensor.") + .AsDispensable() + .AsDuplicable(); + + AddInput("FFNLnScale", "The layer_norm scale of FusedFeedForward op") + .AsDuplicable(); + AddInput("FFNLnBias", "The layer_norm bias of FusedFeedForward op") + .AsDuplicable(); + AddInput("FFN1Weight", "The linear1 weight of FusedFeedForward op") + .AsDuplicable(); + AddInput("FFN1Bias", "The linear1 bias of FusedFeedForward op") + .AsDispensable() + .AsDuplicable(); + AddInput("FFN2Weight", "The linear2 weight of FusedFeedForward op") + .AsDuplicable(); + AddInput("FFN2Bias", "The linear2 bias input of FusedFeedForward op") + .AsDispensable() + .AsDuplicable(); + + AddOutput("CacheKVOut", "The updated cache KV. Inplace with CacheKV") + .AsDispensable() + .AsDuplicable(); + AddOutput("Out", "Result after multi ."); + + AddAttr("pre_layer_norm", + "if true, the attention op uses pre_layer_norm architecure, " + "else, uses post_layer_norm architecuture. " + "[default true].") + .SetDefault(true); + AddAttr("epsilon", + "Constant for numerical stability [default 1e-5].") + .SetDefault(1e-5) + .AddCustomChecker([](const float &epsilon) { + PADDLE_ENFORCE_EQ(epsilon >= 0.0f && epsilon <= 0.001f, true, + platform::errors::InvalidArgument( + "'epsilon' in Op(LayerNorm) should be between" + "0.0 and 0.001, But received [%s].", + epsilon)); + }); + + AddAttr("dropout_rate", "Probability of setting units to zero.") + .SetDefault(.5f) + .AddCustomChecker([](const float &drop_p) { + PADDLE_ENFORCE_EQ(drop_p >= 0.0f && drop_p <= 1.0f, true, + platform::errors::InvalidArgument( + "'dropout_rate' must be between 0.0 and 1.0.")); + }); + + AddAttr("dropout_is_test", + "(bool, default false) Set to true for inference only, false " + "for training. Some layers may run faster when this is true.") + .SetDefault(false); + AddAttr( + "dropout_implementation", + "[\"downgrade_in_infer\"|\"upscale_in_train\"]" + "The meaning is the same as 'attn_dropout_implementation'.") + .SetDefault("downgrade_in_infer") + .AddCustomChecker([](const std::string &type) { + PADDLE_ENFORCE_EQ( + type == "downgrade_in_infer" || type == "upscale_in_train", true, + platform::errors::InvalidArgument( + "dropout_implementation can only be downgrade_in_infer or " + "upscale_in_train")); + }); + AddAttr("act_method", "act_method").SetDefault("gelu"); + + AddAttr( + "ring_id", + "ring id for tensor model parallel. distributed training and inference") + .SetDefault(-1); + + AddComment(R"DOC(fused multi transformer layers op)DOC"); + } +}; + +} // namespace operators +} // namespace paddle + +namespace ops = paddle::operators; +REGISTER_OPERATOR( + fused_multi_transformer, ops::FusedMultiTransformerOp, + ops::FusedMultiTransformerOpOpMaker, + paddle::framework::EmptyGradOpMaker, + paddle::framework::EmptyGradOpMaker); diff --git a/paddle/fluid/operators/fused/fused_multi_transformer_op.cu b/paddle/fluid/operators/fused/fused_multi_transformer_op.cu new file mode 100644 index 0000000000000..e38ac9a0ad2da --- /dev/null +++ b/paddle/fluid/operators/fused/fused_multi_transformer_op.cu @@ -0,0 +1,1343 @@ +/* Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. + * Copyright (c) 2011-2021, NVIDIA CORPORATION. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ +// This file has been adapted from FasterTransformer file: +// https://github.com/NVIDIA/FasterTransformer/blob/v4.0/fastertransformer/cuda/masked_multihead_attention.cu +// We add License in the head. + +#include +#include +#include +#include "paddle/fluid/framework/op_registry.h" +#include "paddle/fluid/framework/operator.h" +#include "paddle/fluid/platform/device/gpu/gpu_device_function.h" +#include "paddle/fluid/platform/device/gpu/gpu_dnn.h" + +#include "paddle/fluid/operators/elementwise/elementwise_add_op.h" +#include "paddle/phi/kernels/funcs/math_function.h" + +#include "paddle/fluid/operators/fused/attention_layer_norm.h" +#include "paddle/fluid/operators/fused/attn_gemm.h" +#include "paddle/fluid/operators/fused/fmha_ref.h" +#include "paddle/fluid/operators/fused/fused_dropout_helper.h" + +#if defined(PADDLE_WITH_NCCL) || defined(PADDLE_WITH_RCCL) +#include "paddle/fluid/platform/collective_helper.h" +#include "paddle/fluid/platform/device/gpu/nccl_helper.h" +#endif + +namespace paddle { +namespace operators { + +using Tensor = framework::Tensor; + +// for debug +// #define _DEBUG_FUSED_MULTI_TRANSFORMER + +template +static void AllReduce(framework::Tensor &tensor, // NOLINT + const int ring_id, + const platform::CUDADeviceContext &ctx) { + if (ring_id == -1) return; +#if defined(PADDLE_WITH_NCCL) || defined(PADDLE_WITH_RCCL) + auto dtype = + platform::ToNCCLDataType(framework::TransToProtoVarType(tensor.dtype())); + int64_t numel = tensor.numel(); + const void *sendbuff = tensor.data(); + auto place = ctx.GetPlace(); + void *recvbuff = tensor.mutable_data(place); + auto comm = platform::NCCLCommContext::Instance().Get(ring_id, place); + auto stream = ctx.stream(); + PADDLE_ENFORCE_GPU_SUCCESS(platform::dynload::ncclAllReduce( + sendbuff, recvbuff, numel, dtype, ncclSum, comm->comm(), stream)); +#else + PADDLE_THROW(platform::errors::Unimplemented( + "PaddlePaddle should compile with NCCL or RCCL when used tensor model " + "parallel op.")); +#endif +} + +namespace { + +namespace plat = paddle::platform; +using float16 = plat::float16; + +#define MMHA_USE_FP32_ACUM_FOR_LOGITS +#define MMHA_USE_FP32_ACUM_FOR_OUT + +template +struct Masked_multihead_attention_params { + // output buffer, [B, 1(seq_len), num_head * dim_head] + T *out; + // qkv_out, [B, 1(seq_len), 3, num_head * dim_head] + const T *qkv; + // bias, [3, num_head, dim_head] + const T *qkv_bias; + // TODO(wangxi): optimize with input_lengths and max_input_len? + // [bsz, 1, 1, time_step(cache_seq_length)+1] + const T *attn_mask; + + // [2, B, num_head, max_seq_len(valid cache_seq_len), dim_head] + // k [B, num_head, dim_head/x, max_seq_len, x], that is `seq_len` first + // v [B, num_head, max_seq_len, dim_head] + T *cache_kv; + + int batch_size; + int num_head; + int timestep; // cache_seq_length + int max_seq_length; + + // 1.f / sqrt(Dh) + float inv_sqrt_dh; +}; + +struct Float8_ { + float2 x; + float2 y; + float2 z; + float2 w; +}; + +// clang-format off + +template struct Qk_vec_ {}; +template <> struct Qk_vec_ { using Type = float; }; +template <> struct Qk_vec_ { using Type = float2; }; +template <> struct Qk_vec_ { using Type = float4; }; +template <> struct Qk_vec_ { using Type = uint32_t; }; +template <> struct Qk_vec_ { using Type = uint32_t; }; +template <> struct Qk_vec_ { using Type = uint2; }; + +template struct K_vec_ {}; +template <> struct K_vec_ { using Type = float; }; +template <> struct K_vec_ { using Type = float2; }; +template <> struct K_vec_ { using Type = float4; }; +template <> struct K_vec_ { using Type = uint32_t; }; +template <> struct K_vec_ { using Type = uint2; }; +template <> struct K_vec_ { using Type = uint4; }; + +template struct V_vec_ {}; +template <> struct V_vec_ { using Type = float; }; +template <> struct V_vec_ { using Type = float2; }; +template <> struct V_vec_ { using Type = float4; }; +template <> struct V_vec_ { using Type = uint32_t; }; +template <> struct V_vec_ { using Type = uint2; }; +template <> struct V_vec_ { using Type = uint4; }; + +#ifdef MMHA_USE_FP32_ACUM_FOR_OUT +template struct V_vec_acum_fp32_ {}; +// template <> struct V_vec_acum_fp32_ { using Type = float; }; +// template <> struct V_vec_acum_fp32_ { using Type = float2; }; +template <> struct V_vec_acum_fp32_ { using Type = float4; }; +// template <> struct V_vec_acum_fp32_ { using Type = float2; }; +// template <> struct V_vec_acum_fp32_ { using Type = Float4_; }; +template <> struct V_vec_acum_fp32_ { using Type = Float8_; }; +#endif + +// clang-format on + +inline __device__ float half_to_float(uint16_t h) { + float f; + asm volatile("cvt.f32.f16 %0, %1;\n" : "=f"(f) : "h"(h)); + return f; +} + +inline __device__ float2 half2_to_float2(uint32_t v) { + uint16_t lo, hi; + asm volatile("mov.b32 {%0, %1}, %2;\n" : "=h"(lo), "=h"(hi) : "r"(v)); + return make_float2(half_to_float(lo), half_to_float(hi)); +} + +inline __device__ uint32_t float2_to_half2(float2 f) { + union { + uint32_t u32; + uint16_t u16[2]; + } tmp; +#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 800 + asm volatile("cvt.rn.f16x2.f32 %0, %1, %2;\n" + : "=r"(tmp.u32) + : "f"(f.y), "f"(f.x)); +#else + asm volatile("cvt.rn.f16.f32 %0, %1;\n" : "=h"(tmp.u16[0]) : "f"(f.x)); + asm volatile("cvt.rn.f16.f32 %0, %1;\n" : "=h"(tmp.u16[1]) : "f"(f.y)); +#endif + return tmp.u32; +} + +inline __device__ float add(float a, float b) { return a + b; } + +inline __device__ float2 add(float2 a, float2 b) { + float2 c; + c.x = add(a.x, b.x); + c.y = add(a.y, b.y); + return c; +} + +inline __device__ float4 add(float4 a, float4 b) { + float4 c; + c.x = add(a.x, b.x); + c.y = add(a.y, b.y); + c.z = add(a.z, b.z); + c.w = add(a.w, b.w); + return c; +} + +inline __device__ uint16_t add(uint16_t a, uint16_t b) { + uint16_t c; + asm volatile("add.f16 %0, %1, %2;\n" : "=h"(c) : "h"(a), "h"(b)); + return c; +} + +inline __device__ uint32_t add(uint32_t a, uint32_t b) { + uint32_t c; + asm volatile("add.f16x2 %0, %1, %2;\n" : "=r"(c) : "r"(a), "r"(b)); + return c; +} + +inline __device__ uint2 add(uint2 a, uint2 b) { + uint2 c; + c.x = add(a.x, b.x); + c.y = add(a.y, b.y); + return c; +} + +inline __device__ uint4 add(uint4 a, uint4 b) { + uint4 c; + c.x = add(a.x, b.x); + c.y = add(a.y, b.y); + c.z = add(a.z, b.z); + c.w = add(a.w, b.w); + return c; +} + +inline __device__ float2 add(uint32_t a, float2 fb) { + float2 fa = half2_to_float2(a); + return add(fa, fb); +} + +inline __device__ Float8_ add(uint4 a, Float8_ fb) { + Float8_ fc; + fc.x = add(a.x, fb.x); + fc.y = add(a.y, fb.y); + fc.z = add(a.z, fb.z); + fc.w = add(a.w, fb.w); + return fc; +} + +template +inline __device__ Acc mul(A a, B b); + +template <> +inline __device__ float mul(float a, float b) { + return a * b; +} + +template <> +inline __device__ float2 mul(float2 a, float2 b) { + float2 c; + c.x = a.x * b.x; + c.y = a.y * b.y; + return c; +} + +template <> +inline __device__ float4 mul(float4 a, float4 b) { + float4 c; + c.x = a.x * b.x; + c.y = a.y * b.y; + c.z = a.z * b.z; + c.w = a.w * b.w; + return c; +} + +template <> +inline __device__ uint16_t mul(uint16_t a, uint16_t b) { + uint16_t c; + asm volatile("mul.f16 %0, %1, %2;\n" : "=h"(c) : "h"(a), "h"(b)); + return c; +} + +template <> +inline __device__ uint32_t mul(uint32_t a, uint32_t b) { + uint32_t c; + asm volatile("mul.f16x2 %0, %1, %2;\n" : "=r"(c) : "r"(a), "r"(b)); + return c; +} + +template <> +inline __device__ uint2 mul(uint2 a, uint2 b) { + uint2 c; + c.x = mul(a.x, b.x); + c.y = mul(a.y, b.y); + return c; +} + +template <> +inline __device__ uint4 mul(uint4 a, uint4 b) { + uint4 c; + c.x = mul(a.x, b.x); + c.y = mul(a.y, b.y); + c.z = mul(a.z, b.z); + c.w = mul(a.w, b.w); + return c; +} + +inline __device__ float sum(float v) { return v; } +inline __device__ float sum(float2 v) { return v.x + v.y; } +inline __device__ float sum(float4 v) { return v.x + v.y + v.z + v.w; } +inline __device__ float sum(uint16_t v) { return half_to_float(v); } +inline __device__ float sum(uint32_t v) { + float2 tmp = half2_to_float2(v); + return tmp.x + tmp.y; +} + +inline __device__ float sum(uint2 v) { + uint32_t c = add(v.x, v.y); + return sum(c); +} + +inline __device__ float sum(uint4 v) { + uint32_t c = add(v.x, v.y); + c = add(c, v.z); + c = add(c, v.w); + return sum(c); +} + +template +inline __device__ float dot(T a, T b) { + return sum(mul(a, b)); +} + +template +inline __device__ float dot(T a, T b) { + return sum(mul(a, b)); +} + +inline __device__ constexpr uint32_t shfl_mask(int threads) { + return threads == 32 ? uint32_t(-1) : (1u << threads) - 1u; +} + +template +inline __device__ __host__ T div_up(T m, T n) { + return (m + n - 1) / n; +} + +inline __device__ float fma(float a, float b, float c) { return a * b + c; } + +inline __device__ float2 fma(float2 a, float2 b, float2 c) { + float2 d; + d.x = fma(a.x, b.x, c.x); + d.y = fma(a.y, b.y, c.y); + return d; +} + +inline __device__ float4 fma(float4 a, float4 b, float4 c) { + float4 d; + d.x = fma(a.x, b.x, c.x); + d.y = fma(a.y, b.y, c.y); + d.z = fma(a.z, b.z, c.z); + d.w = fma(a.w, b.w, c.w); + return d; +} + +inline __device__ uint32_t fma(uint32_t a, uint32_t b, uint32_t c) { + uint32_t d; + asm volatile("fma.rn.f16x2 %0, %1, %2, %3;\n" + : "=r"(d) + : "r"(a), "r"(b), "r"(c)); + return d; +} + +inline __device__ uint2 fma(uint2 a, uint2 b, uint2 c) { + uint2 d; + d.x = fma(a.x, b.x, c.x); + d.y = fma(a.y, b.y, c.y); + return d; +} + +inline __device__ uint4 fma(uint4 a, uint4 b, uint4 c) { + uint4 d; + d.x = fma(a.x, b.x, c.x); + d.y = fma(a.y, b.y, c.y); + d.z = fma(a.z, b.z, c.z); + d.w = fma(a.w, b.w, c.w); + return d; +} + +inline __device__ float2 fma(float a, float2 b, float2 c) { + float2 d; + d.x = fma(a, b.x, c.x); + d.y = fma(a, b.y, c.y); + return d; +} + +inline __device__ float4 fma(float a, float4 b, float4 c) { + float4 d; + d.x = fma(a, b.x, c.x); + d.y = fma(a, b.y, c.y); + d.z = fma(a, b.z, c.z); + d.w = fma(a, b.w, c.w); + return d; +} + +inline __device__ Float8_ fma(float a, Float8_ b, Float8_ c) { + Float8_ d; + d.x = fma(a, b.x, c.x); + d.y = fma(a, b.y, c.y); + d.z = fma(a, b.z, c.z); + d.w = fma(a, b.w, c.w); + return d; +} + +inline __device__ uint32_t h0_h0(uint16_t a) { + uint32_t b; + asm volatile("mov.b32 %0, {%1, %1};" : "=r"(b) : "h"(a)); + return b; +} + +inline __device__ uint32_t fma(uint16_t a, uint32_t b, uint32_t c) { + return fma(h0_h0(a), b, c); +} + +inline __device__ uint2 fma(uint16_t a, uint2 b, uint2 c) { + uint32_t s = h0_h0(a); + uint2 d; + d.x = fma(s, b.x, c.x); + d.y = fma(s, b.y, c.y); + return d; +} + +inline __device__ uint4 fma(uint16_t a, uint4 b, uint4 c) { + uint32_t s = h0_h0(a); + uint4 d; + d.x = fma(s, b.x, c.x); + d.y = fma(s, b.y, c.y); + d.z = fma(s, b.z, c.z); + d.w = fma(s, b.w, c.w); + return d; +} + +inline __device__ float cast_to_float(float u) { return u; } + +inline __device__ float2 cast_to_float(float2 u) { return u; } + +inline __device__ float4 cast_to_float(float4 u) { return u; } + +inline __device__ Float8_ cast_to_float(uint4 u) { + Float8_ tmp; + tmp.x = half2_to_float2(u.x); + tmp.y = half2_to_float2(u.y); + tmp.z = half2_to_float2(u.z); + tmp.w = half2_to_float2(u.w); + return tmp; +} + +template +inline __device__ float qk_dot_(const K_vec (&q)[N], const K_vec (&k)[N]) { + K_vec qk_vec = mul(q[0], k[0]); +#pragma unroll + for (int ii = 1; ii < N; ++ii) { + qk_vec = fma(q[ii], k[ii], qk_vec); + } + + float qk = sum(qk_vec); +#pragma unroll + for (int mask = THREADS_PER_KEY / 2; mask >= 1; mask /= 2) { + qk += __shfl_xor_sync(uint32_t(-1), qk, mask); + } + return qk; +} + +template +struct Qk_dot { + template + static inline __device__ float dot(const K_vec (&q)[N], const K_vec (&k)[N]) { + return qk_dot_(q, k); + } +}; + +template +inline __device__ float block_sum(float *red_smem, float sum) { + int warp = threadIdx.x / WARP_SIZE; + int lane = threadIdx.x % WARP_SIZE; + +#pragma unroll + for (int mask = WARP_SIZE / 2; mask >= 1; mask /= 2) { + sum += __shfl_xor_sync(uint32_t(-1), sum, mask); + } + + if (lane == 0) { + red_smem[warp] = sum; + } + __syncthreads(); + + if (lane < WARPS_PER_BLOCK) { + sum = red_smem[lane]; + } + +#pragma unroll + for (int mask = WARPS_PER_BLOCK / 2; mask >= 1; mask /= 2) { + sum += __shfl_xor_sync(uint32_t(-1), sum, mask); + } + + return __shfl_sync(uint32_t(-1), sum, 0); +} + +inline __device__ void convert_from_float(float &dst, float src) { // NOLINT + dst = src; +} + +inline __device__ void convert_from_float(float4 &dst, float4 src) { // NOLINT + dst = src; +} + +inline __device__ void convert_from_float(plat::float16 &dst, // NOLINT + float src) { + dst = static_cast(src); +} + +inline __device__ void convert_from_float(uint4 &dst, Float8_ src) { // NOLINT + dst.x = float2_to_half2(src.x); + dst.y = float2_to_half2(src.y); + dst.z = float2_to_half2(src.z); + dst.w = float2_to_half2(src.w); +} + +inline __device__ void zero(uint16_t &dst) { dst = uint16_t(0); } // NOLINT + +template +inline __device__ void zero(T &dst) { // NOLINT + constexpr int WORDS = sizeof(T) / 4; + union { + T raw; + uint32_t words[WORDS]; + } tmp; +#pragma unroll + for (int ii = 0; ii < WORDS; ++ii) { + tmp.words[ii] = 0u; + } + dst = tmp.raw; +} + +template +__global__ void masked_multihead_attention_kernel( + Masked_multihead_attention_params params) { +#if CUDA_ARCH_FP16_SUPPORTED(__CUDA_ARCH__) + + static_assert(Dh % THREADS_PER_KEY == 0, ""); + static_assert(Dh % THREADS_PER_VALUE == 0, ""); + + constexpr int WARP_SIZE = 32; + constexpr int WARPS_PER_BLOCK = THREADS_PER_BLOCK / WARP_SIZE; + + extern __shared__ char smem_[]; + + float *qk_smem = reinterpret_cast(smem_); + + char *logits_smem_ = smem_; + // fp32 accum for logits + float *logits_smem = reinterpret_cast(logits_smem_); + + T *out_smem = reinterpret_cast(smem_); + + __shared__ float red_smem[WARPS_PER_BLOCK * 2]; + __shared__ T q_smem[Dh]; + + const int bi = blockIdx.y; + const int hi = blockIdx.x; + const int bhi = bi * params.num_head + hi; + const int tid = threadIdx.x; + + float qk_max = -FLT_MAX; + + // qkv [B, S=1, 3, num_head, head_dim] + int qkv_base_offset = bi * 3 * params.num_head * Dh + hi * Dh; + + using Qk_vec = typename Qk_vec_::Type; + constexpr int QK_VEC_SIZE = sizeof(Qk_vec) / sizeof(T); + static_assert(Dh % QK_VEC_SIZE == 0 && Dh / QK_VEC_SIZE <= WARP_SIZE, ""); + constexpr int QK_VECS_PER_WARP = Dh / QK_VEC_SIZE; + + // cache_k, [B, num_head, head_dim / x, max_seq_len, x] + // x == 4/8 for FP32/FP16, 128bit, 16Byte + constexpr int QK_ELTS_IN_16B = 16 / sizeof(T); + constexpr int QK_VECS_IN_16B = 16 / sizeof(Qk_vec); + + const T *q_base = params.qkv; + const T *k_base = params.qkv + params.num_head * Dh; + const T *q_bias_base = params.qkv_bias; + const T *k_bias_base = params.qkv_bias + params.num_head * Dh; + + if (tid < QK_VECS_PER_WARP) { + int qk_offset = qkv_base_offset + tid * QK_VEC_SIZE; + int qk_bias_offset = hi * Dh + tid * QK_VEC_SIZE; + + Qk_vec q = *reinterpret_cast(&q_base[qk_offset]); + Qk_vec k = *reinterpret_cast(&k_base[qk_offset]); + + Qk_vec q_bias = + *reinterpret_cast(&q_bias_base[qk_bias_offset]); + Qk_vec k_bias = + *reinterpret_cast(&k_bias_base[qk_bias_offset]); + + q = add(q, q_bias); + // TODO(wangxi): See this https://github.com/microsoft/unilm/issues/510 + // we may not require k_bias. + k = add(k, k_bias); + + *reinterpret_cast(&q_smem[tid * QK_VEC_SIZE]) = q; + + int co = tid / QK_VECS_IN_16B; + int ci = (tid % QK_VECS_IN_16B) * QK_VEC_SIZE; + int offset = bhi * params.max_seq_length * Dh + + co * params.max_seq_length * QK_ELTS_IN_16B + + params.timestep * QK_ELTS_IN_16B + ci; + *reinterpret_cast(¶ms.cache_kv[offset]) = k; + + float qk = dot(q, k); +#pragma unroll + for (int mask = QK_VECS_PER_WARP / 2; mask >= 1; mask /= 2) { + qk += __shfl_xor_sync(shfl_mask(QK_VECS_PER_WARP), qk, mask); + } + + qk *= params.inv_sqrt_dh; + if (tid == 0) { + // NOTE(wangxi): mask must be 0.0 + // T mask = params.attn_mask[ + // bi * (params.timestep + 1) + params.timestep]; + // qk += static_cast(mask); + qk_max = qk; + qk_smem[params.timestep] = qk; + } + } + __syncthreads(); + +#ifdef _DEBUG_FUSED_MULTI_TRANSFORMER + if (bi == 0 && hi == 0 && tid == 0) { + printf("=======q_out=======\n"); + for (int i = 0; i < Dh; ++i) printf("%f ", static_cast(q_smem[i])); + printf("\n"); + } + __syncthreads(); +#endif + + using K_vec = typename K_vec_::Type; + constexpr int K_VEC_SIZE = sizeof(K_vec) / sizeof(T); + static_assert(Dh % K_VEC_SIZE == 0, ""); + constexpr int K_ELTS_PER_THREAD = Dh / THREADS_PER_KEY; + constexpr int K_VECS_PER_THREAD = K_ELTS_PER_THREAD / K_VEC_SIZE; + + int ko = tid / THREADS_PER_KEY; + int ki = (tid % THREADS_PER_KEY) * K_VEC_SIZE; + + K_vec q[K_VECS_PER_THREAD]; +#pragma unroll + for (int i = 0; i < K_VECS_PER_THREAD; ++i) { + q[i] = *reinterpret_cast( + &q_smem[ki + i * THREADS_PER_KEY * K_VEC_SIZE]); + } + + constexpr int K_PER_ITER = THREADS_PER_BLOCK / THREADS_PER_KEY; + constexpr int K_PER_WARP = WARP_SIZE / THREADS_PER_KEY; + + T *k_cache = ¶ms.cache_kv[bhi * params.max_seq_length * Dh + ki]; + int ti_end = div_up(params.timestep, K_PER_WARP) * K_PER_WARP; + + for (int ti = ko; ti < ti_end; ti += K_PER_ITER) { + K_vec k[K_VECS_PER_THREAD]; +#pragma unroll + for (int ii = 0; ii < K_VECS_PER_THREAD; ++ii) { + int jj = ii * params.max_seq_length + ti; + if (ti < params.timestep) { + k[ii] = *reinterpret_cast(&k_cache[jj * QK_ELTS_IN_16B]); + } + } + + float qk = Qk_dot::dot(q, k) * params.inv_sqrt_dh; + + // bool is_mask = false; + if (ti < params.timestep && tid % THREADS_PER_KEY == 0) { + // qk_max = is_mask ? qk_max : fmaxf(qk_max, qk); + T mask = params.attn_mask[bi * (params.timestep + 1) + ti]; + qk += static_cast(mask); + qk_max = fmaxf(qk_max, qk); + + qk_smem[ti] = qk; + } + } + +#pragma unroll + for (int mask = WARP_SIZE / 2; mask >= THREADS_PER_KEY; mask /= 2) { + qk_max = fmaxf(qk_max, __shfl_xor_sync(uint32_t(-1), qk_max, mask)); + } + + const int warp = tid / WARP_SIZE; + const int lane = tid % WARP_SIZE; + + if (lane == 0) { + red_smem[warp] = qk_max; + } + + __syncthreads(); + + qk_max = lane < WARPS_PER_BLOCK ? red_smem[lane] : -FLT_MAX; +#pragma unroll + for (int mask = WARPS_PER_BLOCK / 2; mask >= 1; mask /= 2) { + qk_max = fmaxf(qk_max, __shfl_xor_sync(uint32_t(-1), qk_max, mask)); + } + + qk_max = __shfl_sync(uint32_t(-1), qk_max, 0); + +#ifdef _DEBUG_FUSED_MULTI_TRANSFORMER + if (bi == 0 && hi == 0 && tid == 0) { + printf("=======qk_out=======\n"); + for (int i = 0; i <= params.timestep; ++i) printf("%f ", qk_smem[i]); + printf("qk_max=%f\n", qk_max); + } + __syncthreads(); +#endif + + float sum = 0.f; + for (int ti = tid; ti <= params.timestep; ti += THREADS_PER_BLOCK) { + // bool is_mask = false; + // float logit = is_mask ? 0.f : __expf(qk_smem[ti] - qk_max); + float logit = __expf(qk_smem[ti] - qk_max); + sum += logit; + qk_smem[ti] = logit; + } + + sum = block_sum(&red_smem[WARPS_PER_BLOCK], sum); + + // FIXME(wangxi): need add 1.e-6f? + float inv_sum = __fdividef(1.f, sum + 1.e-6f); + for (int ti = tid; ti <= params.timestep; ti += THREADS_PER_BLOCK) { + convert_from_float(logits_smem[ti], qk_smem[ti] * inv_sum); + } + __syncthreads(); + + constexpr int V_VEC_SIZE = Dh / THREADS_PER_VALUE; + using V_vec = typename V_vec_::Type; + + int vo = tid / THREADS_PER_VALUE; + int vi = (tid % THREADS_PER_VALUE) * V_VEC_SIZE; + + T *v_cache = ¶ms.cache_kv[params.batch_size * params.num_head * + params.max_seq_length * Dh + + bhi * params.max_seq_length * Dh + vi]; + +#ifdef MMHA_USE_FP32_ACUM_FOR_OUT + using V_vec_acum = typename V_vec_acum_fp32_::Type; +#else + using V_vec_acum = V_vec; +#endif + + V_vec_acum out; + zero(out); + + constexpr int V_PER_ITER = THREADS_PER_BLOCK / THREADS_PER_VALUE; + for (int ti = vo; ti < params.timestep; ti += V_PER_ITER) { + V_vec v = *reinterpret_cast(&v_cache[ti * Dh]); +#if defined(MMHA_USE_FP32_ACUM_FOR_LOGITS) + float logit = logits_smem[ti]; + out = fma(logit, cast_to_float(v), out); +#else + T logit = logits_smem[ti]; + // Update the partial sums. + out = fma(logit, v, out); +#endif + } + +#ifdef _DEBUG_FUSED_MULTI_TRANSFORMER + if (bi == 0 && hi == 0 && tid == 0) { + printf("======logits_out=====\n"); + for (int i = 0; i <= params.timestep; ++i) printf("%f ", logits_smem[i]); + printf("\n"); + } + __syncthreads(); +#endif + + if (vo == (params.timestep % V_PER_ITER)) { + V_vec v = *reinterpret_cast( + ¶ms.qkv[2 * params.num_head * Dh + qkv_base_offset + vi]); + V_vec v_bias = *reinterpret_cast( + ¶ms.qkv_bias[2 * params.num_head * Dh + hi * Dh + vi]); + v = add(v, v_bias); + *reinterpret_cast(&v_cache[params.timestep * Dh]) = v; + +#if defined(MMHA_USE_FP32_ACUM_FOR_LOGITS) + out = fma(logits_smem[params.timestep], cast_to_float(v), out); +#else + out = fma(logits_smem[params.timestep], v, out); +#endif + } + + __syncthreads(); + +#pragma unroll + for (int active_groups = V_PER_ITER; active_groups >= 2; active_groups /= 2) { + int midpoint = active_groups / 2; + + if (vo >= midpoint && vo < active_groups) { +#ifdef MMHA_USE_FP32_ACUM_FOR_OUT + convert_from_float( + *reinterpret_cast(&out_smem[(vo - midpoint) * Dh + vi]), + out); +#else + *reinterpret_cast(&out_smem[(vo - midpoint) * Dh + vi]) = out; +#endif + } + __syncthreads(); + if (vo < midpoint) { + out = add(*reinterpret_cast(&out_smem[vo * Dh + vi]), out); + } + __syncthreads(); + } + + if (vo == 0) { +#ifdef MMHA_USE_FP32_ACUM_FOR_OUT + convert_from_float(*reinterpret_cast(¶ms.out[bhi * Dh + vi]), + out); +#else + *reinterpret_cast(¶ms.out[bhi * Dh + vi]) = out; +#endif + } + +#ifdef _DEBUG_FUSED_MULTI_TRANSFORMER + __syncthreads(); + if (bi == 0 && hi == 0 && tid == 0) { + printf("======fmha_out=====\n"); + for (int i = 0; i < Dh; ++i) + printf("%f ", static_cast(params.out[i])); + printf("\n"); + } +#endif +#else + assert(false); +#endif +} + +template +inline size_t smem_size_in_bytes( + const Masked_multihead_attention_params ¶ms, int dim_head, + int threads_per_value, int threads_per_block) { + size_t qk_sz = div_up(params.timestep + 1, 4) * 16; + size_t logits_sz = 0; + +#ifndef MMHA_USE_FP32_ACUM_FOR_LOGITS + if (sizeof(T) != 4) { + logits_sz = div_up(params.max_seq_length, 4) * 4 * sizeof(T); + } +#endif + size_t softmax_sz = qk_sz + logits_sz; + + int rows_per_red = threads_per_block / threads_per_value; + size_t red_sz = rows_per_red * dim_head * sizeof(T) / 2; + + return max(softmax_sz, red_sz); +} + +#define MMHA_LAUNCH_KERNEL(T, Dh, THDS_PER_KEY, THDS_PER_VALUE, \ + THDS_PER_BLOCK, stream) \ + size_t smem_sz = \ + smem_size_in_bytes(params, Dh, THDS_PER_VALUE, THDS_PER_BLOCK); \ + dim3 grid(params.num_head, params.batch_size); \ + masked_multihead_attention_kernel< \ + T, Dh, THDS_PER_KEY, THDS_PER_VALUE, \ + THDS_PER_BLOCK><<>>(params) + +template +void fmha_launch_kernel(const Masked_multihead_attention_params ¶ms, + const cudaStream_t &stream) { + constexpr int THREADS_PER_VALUE = Dh * sizeof(T) / 16; + if (params.timestep < 32) { + MMHA_LAUNCH_KERNEL(T, Dh, 4, THREADS_PER_VALUE, 64, stream); + } else if (params.timestep < 2048) { + MMHA_LAUNCH_KERNEL(T, Dh, 2, THREADS_PER_VALUE, 128, stream); + } else { + MMHA_LAUNCH_KERNEL(T, Dh, 1, THREADS_PER_VALUE, 256, stream); + } +} + +template +void fmha(const platform::CUDADeviceContext &dev_ctx, const Tensor &qkv_tensor, + const Tensor &qkv_bias_tensor, const Tensor &src_mask_tensor, + Tensor *cache_kv_tensor, Tensor *out_tensor, int batch_size, + int max_seq_length, int num_head, int dim_head, int timestep, + float inv_sqrt_dh) { + Masked_multihead_attention_params params; + params.out = out_tensor->data(); + params.qkv = qkv_tensor.data(); + params.qkv_bias = qkv_bias_tensor.data(); + params.attn_mask = src_mask_tensor.data(); + params.cache_kv = cache_kv_tensor->data(); + + params.batch_size = batch_size; + params.num_head = num_head; + params.timestep = timestep; + params.max_seq_length = max_seq_length; + params.inv_sqrt_dh = inv_sqrt_dh; + + switch (dim_head) { + case 32: + fmha_launch_kernel(params, dev_ctx.stream()); + break; + case 64: + fmha_launch_kernel(params, dev_ctx.stream()); + break; + case 128: + fmha_launch_kernel(params, dev_ctx.stream()); + break; + default: + PADDLE_THROW(platform::errors::Unimplemented( + "dim_head = %d is unsupport, only support " + "dim_head = 32, 64 or 128 for now.", + dim_head)); + } +} + +// NOTE: simd with 16Bytes(128bit), float is 4, float16 is 8 +constexpr int VEC_16B = 16; + +template +__global__ void write_cache_k_kernel(T *cache_k, const T *k, const int num_head, + const int dim_head, const int seq_len, + const int max_seq_len) { + const int bi = blockIdx.y; + const int hi = blockIdx.z; + constexpr int X_ELEMS = VEC_16B / sizeof(T); + + // [bsz, num_head, seq_len, dim_head/x, x] + auto k_src = reinterpret_cast( + k + bi * num_head * seq_len * dim_head + hi * seq_len * dim_head); + // [bsz, num_head, dim_head/x, max_seq_len, x] + auto k_dst = reinterpret_cast( + cache_k + bi * num_head * max_seq_len * dim_head + + hi * max_seq_len * dim_head); + + const int out_idx = blockIdx.x * blockDim.x + threadIdx.x; + // vec size + int dim_head_div_x = dim_head / X_ELEMS; + + // FIXME(wangxi): num_head is not need? + // if (out_idx >= num_head * dim_head_div_x * max_seq_len) return; + if (out_idx >= dim_head_div_x * max_seq_len) return; + + int idx = out_idx; + const int k_seq_len_id = idx % max_seq_len; + // idx = (idx - k_seq_len_id) / max_seq_len; + idx = idx / max_seq_len; + const int k_vec_id = idx % dim_head_div_x; + + if (k_seq_len_id < seq_len) { + k_dst[out_idx] = k_src[k_seq_len_id * dim_head_div_x + k_vec_id]; + } +} + +template +__global__ void write_cache_v_kernel(T *cache_v, const T *v, const int num_head, + const int dim_head, const int seq_len, + const int max_seq_len) { + const int bi = blockIdx.y; + const int hi = blockIdx.z; + + // [bsz, num_head, seq_len, dim_head/x, x] + auto v_src = reinterpret_cast( + v + bi * num_head * seq_len * dim_head + hi * seq_len * dim_head); + // [bsz, num_head, max_seq_len, dim_head/x, x] + auto v_dst = reinterpret_cast( + cache_v + bi * num_head * max_seq_len * dim_head + + hi * max_seq_len * dim_head); + + const int idx = blockIdx.x * blockDim.x + threadIdx.x; + constexpr int X_ELEMS = VEC_16B / sizeof(T); + const int dim_head_div_x = dim_head / X_ELEMS; + + if (idx >= dim_head_div_x * seq_len) return; + + v_dst[idx] = v_src[idx]; +} + +template +void write_cache_kv(const platform::CUDADeviceContext &dev_ctx, T *cache_k, + T *cache_v, const T *k, const T *v, const int bsz, + const int num_head, const int seq_len, + const int max_seq_len, const int dim_head) { + constexpr int block_sz = 128; + constexpr int x = VEC_16B / sizeof(T); + + assert(dim_head % x == 0); + PADDLE_ENFORCE_EQ( + dim_head % x, 0, + platform::errors::PreconditionNotMet( + "dim_head=%d must be divisible by vec_size=%d", dim_head, x)); + + int max_size = max_seq_len * dim_head / x; + int size = seq_len * dim_head / x; + dim3 grid(div_up(max_size, block_sz), bsz, num_head); + dim3 grid_v(div_up(size, block_sz), bsz, num_head); + + // transpose [bsz, num_head, seq_len, dim_head/x, x]-> + // [bsz, num_head, dim_head/x, max_seq_len, x] + write_cache_k_kernel<<>>( + cache_k, k, num_head, dim_head, seq_len, max_seq_len); + + // copy [bsz, num_head, seq_len, dim_head/x, x]-> + // [bsz, num_head, max_seq_len, dim_head/x, x] + write_cache_v_kernel<<>>( + cache_v, v, num_head, dim_head, seq_len, max_seq_len); +} + +} // namespace + +template +class FusedMultiTransformerOpKernel : public framework::OpKernel { + public: + void Compute(const framework::ExecutionContext &ctx) const override { + using U = LayerNormParamType; + auto place = ctx.GetPlace(); + auto &dev_ctx = ctx.cuda_device_context(); + + auto *time_step = ctx.Input("TimeStep"); + // 0. input + auto *input_x = ctx.Input("X"); + const auto input_x_dims = input_x->dims(); + int bsz = input_x_dims[0]; + int seq_len = input_x_dims[1]; + int dim_embed = input_x_dims[2]; + int bsz_seq = bsz * seq_len; + + // 1. layer norm + const auto pre_layer_norm = ctx.Attr("pre_layer_norm"); + const float epsilon = ctx.Attr("epsilon"); + auto ln_scales = ctx.MultiInput("LnScale"); + auto ln_biases = ctx.MultiInput("LnBias"); + + auto ln_compute = AttnLayerNorm(dev_ctx, epsilon, bsz_seq, dim_embed); + Tensor ln_mean, ln_var; + auto *ln_mean_data = ln_mean.mutable_data({bsz_seq}, place); + auto *ln_var_data = ln_var.mutable_data({bsz_seq}, place); + + // 2. qkv + // x: qkv's input [batch_size, seq_len, dim_embed] + // y: qkv's weight: [3, num_head, dim_head, dim_embed] + auto qkv_weights = ctx.MultiInput("QKVW"); + auto qkv_biases = ctx.MultiInput("QKVBias"); + const auto qkv_w_dims = qkv_weights[0]->dims(); + int num_head = qkv_w_dims[1]; + int dim_head = qkv_w_dims[2]; + int hidden_size = num_head * dim_head; + int output_size = 3 * hidden_size; + int input_size = dim_embed; + + bool compute_bias = qkv_biases.size() > 0 && time_step == nullptr; + // (transA, transB, compute_bias) = (false, true, false) + auto qkv_compute = AttnMatMul(dev_ctx, false, true, bsz_seq, output_size, + input_size, compute_bias); + Tensor qkv_out; + auto *qkv_out_data = + qkv_out.mutable_data({bsz, seq_len, 3, num_head, dim_head}, place); + + // 3. fmha + AttnDropoutParam attn_param(true, "upscale_in_train", 0.0, true, true, 0, + nullptr); + auto fmha_compute = + FMHARef(dev_ctx, bsz, seq_len, num_head, dim_head, attn_param); + auto *src_mask = ctx.Input("SrcMask"); + auto cache_kvs = ctx.MultiInput("CacheKV"); + auto cache_kv_outs = ctx.MultiOutput("CacheKVOut"); + // auto *time_step = ctx.Input("TimeStep"); + + auto out_seq_len = seq_len; + if (time_step) { + PADDLE_ENFORCE_EQ(time_step->place(), platform::CPUPlace(), + platform::errors::PreconditionNotMet( + "The place of input(TimeStep) must be CPUPlace.")); + // cache_seq_len + int time_step_value = time_step->data()[0]; + PADDLE_ENFORCE_GT(time_step_value, 0, + platform::errors::PreconditionNotMet( + "The value of time_step must > 0, but now is %d", + time_step_value)); + PADDLE_ENFORCE_EQ( + seq_len, 1, + platform::errors::PreconditionNotMet( + "In decode stage, the seq_len of input must be 1, but now is %d", + seq_len)); + out_seq_len += time_step_value; + } + + Tensor transpose_out_2, qk_out; + auto *transpose_out_2_data = transpose_out_2.mutable_data( + {3, bsz, num_head, seq_len, dim_head}, place); + auto *qk_out_data = + qk_out.mutable_data({bsz, num_head, seq_len, out_seq_len}, place); + + Tensor src_mask_out, softmax_out; + Tensor attn_dropout_mask_out, attn_dropout_out; + Tensor qktv_out, fmha_out; + auto *src_mask_out_data = src_mask_out.mutable_data( + {bsz, num_head, seq_len, out_seq_len}, place); + auto *softmax_out_data = softmax_out.mutable_data( + {bsz, num_head, seq_len, out_seq_len}, place); + + auto *attn_dropout_mask_out_data = attn_dropout_mask_out.mutable_data( + {bsz, num_head, seq_len, out_seq_len}, place); + auto *attn_dropout_data_data = attn_dropout_out.mutable_data( + {bsz, num_head, seq_len, out_seq_len}, place); + + auto *qktv_out_data = + qktv_out.mutable_data({bsz, num_head, seq_len, dim_head}, place); + auto *fmha_out_data = + fmha_out.mutable_data({bsz, seq_len, num_head, dim_head}, place); + + // 4. out_linear + auto out_linear_weights = ctx.MultiInput("OutLinearW"); + auto out_linear_biases = ctx.MultiInput("OutLinearBias"); + int ring_id = ctx.Attr("ring_id"); + // (transA, transB, compute_bias) = (false, false, false) + auto out_linear_compute = AttnMatMul(dev_ctx, false, false, bsz_seq, + dim_embed, hidden_size, false); + + // 5. ln(residual + bias) + DropoutParam dropout_param2(true, 0, true, true, 0.0, nullptr, 0); + FusedDropoutLayerNormHelper fused_dropout_layernorm_helper( + dev_ctx, bsz_seq, dim_embed, dropout_param2, epsilon); + auto ffn_ln_scales = ctx.MultiInput("FFNLnScale"); + auto ffn_ln_biases = ctx.MultiInput("FFNLnBias"); + Tensor bias_dropout_residual_out, dropout_mask_out; + auto *bias_dropout_residual_out_data = + bias_dropout_residual_out.mutable_data({bsz, seq_len, dim_embed}, + place); + auto *dropout_mask_out_data = dropout_mask_out.mutable_data( + {bsz, seq_len, dim_embed}, place); + + // 6. ffn matmul1 + auto ffn1_weights = ctx.MultiInput("FFN1Weight"); + auto ffn1_biases = ctx.MultiInput("FFN1Bias"); + auto ffn1_weight_dim = ffn1_weights[0]->dims(); + + int dim_ffn = ffn1_weight_dim[1]; + auto ffn1_linear_compute = AttnMatMul(dev_ctx, false, false, bsz_seq, + dim_ffn, dim_embed, false); + Tensor ffn1_out; + auto *ffn1_out_data = ffn1_out.mutable_data({bsz_seq, dim_ffn}, place); + + // 7. ffn act + bias + DropoutParam ffn1_dropout_param(true, 0, true, true, 0.0, nullptr, 0); + FusedDropoutHelper fused_act_dropout_helper( + dev_ctx, bsz_seq, dim_ffn, ffn1_dropout_param); + Tensor ffn1_dropout_out, ffn1_dropout_mask; + auto *ffn1_dropout_out_data = + ffn1_dropout_out.mutable_data({bsz_seq, dim_ffn}, place); + auto *ffn1_dropout_mask_data = + ffn1_dropout_mask.mutable_data({bsz_seq, dim_ffn}, place); + + // 8. ffn2 matmul + auto ffn2_weights = ctx.MultiInput("FFN2Weight"); + auto ffn2_biases = ctx.MultiInput("FFN2Bias"); + auto ffn2_linear_compute = AttnMatMul(dev_ctx, false, false, bsz_seq, + dim_embed, dim_ffn, false); + + // 9. ffn2 residual bias + DropoutParam ffn2_dropout_param(true, 0, true, true, 0.0, nullptr, 0); + FusedDropoutLayerNormHelper ffn2_fused_dropout_helper( + dev_ctx, bsz_seq, dim_embed, ffn2_dropout_param, epsilon); + + // calc + auto *out = ctx.Output("Out"); + auto *from_data = out->mutable_data(place); + Tensor *from_tensor = out; + Tensor tmp_out; + auto *tmp_out_data = + tmp_out.mutable_data({bsz, seq_len, dim_embed}, place); + + auto *x_data = input_x->data(); + Tensor *buf0 = nullptr; + Tensor *buf1 = nullptr; + + // step0: x --> buf1 + // step1: buf1 --> buf0 + // step2: buf0 --> buf1 + int layers = qkv_weights.size(); + if (layers & 1) { + // odd, set buf1 as out + buf0 = &tmp_out; + buf1 = out; + } else { + // even, set buf0 as out + buf0 = out; + buf1 = &tmp_out; + } + + for (int i = 0; i < layers; ++i) { + // step1. layer_norm + if (i == 0 && pre_layer_norm) { + auto *ln_scale_data = ln_scales[i]->data(); + auto *ln_bias_data = ln_biases[i]->data(); + // TODO(wangxi): can remove mean var in inference + ln_compute.ComputeForward(x_data, ln_scale_data, ln_bias_data, + buf1->data(), ln_mean_data, ln_var_data); + } else if (!pre_layer_norm) { + PADDLE_THROW(platform::errors::Unimplemented( + "Unimplemented post_layer_norm for now.")); + } +#ifdef _DEBUG_FUSED_MULTI_TRANSFORMER + VLOG(0) << "step1"; +#endif + + // step2. qkv + const Tensor *qkv_bias = qkv_biases.size() > 0 ? qkv_biases[i] : nullptr; + // NOTE: in decoder stage, bias is fused in fmha + const Tensor *bias = time_step ? nullptr : qkv_bias; + qkv_compute.ComputeForward(qkv_weights[i], buf1, bias, &qkv_out, + &qkv_out); +#ifdef _DEBUG_FUSED_MULTI_TRANSFORMER + VLOG(0) << "step2"; +#endif + + // step3. fmha + const Tensor *cache_kv = cache_kvs.size() > 0 ? cache_kvs[i] : nullptr; + Tensor *cache_kv_out = cache_kv ? cache_kv_outs[i] : nullptr; + + if (time_step) { // generation decoder stage + // [2, batch_size, num_head, max_seq_len, head_size] + int max_seq_len = cache_kv->dims()[3]; + fmha(dev_ctx, qkv_out, *qkv_bias, *src_mask, cache_kv_out, &fmha_out, + bsz, max_seq_len, num_head, dim_head, time_step->data()[0], + 1. / sqrt(dim_head)); + } else if (cache_kv_out) { // generation context stage + // TODO(wangxi): can remove dropout in inference + fmha_compute.ComputeForward( + qkv_out, nullptr, src_mask, &transpose_out_2, nullptr, &qk_out, + &src_mask_out, &softmax_out, &attn_dropout_mask_out, + &attn_dropout_out, &qktv_out, &fmha_out); + // [3, bsz, num_head, seq_len, head_dim] + T *qkv_data = transpose_out_2_data; + int64_t q_size = bsz * seq_len * num_head * dim_head; + int64_t k_size = q_size; + const T *q_ptr = qkv_data; + const T *k_ptr = q_ptr + q_size; + const T *v_ptr = k_ptr + k_size; + + // [2, bsz, num_head, max_seq_len, head_dim] + int max_seq_len = cache_kv_out->dims()[3]; + T *cache_kv_data = cache_kv_out->data(); + int64_t cache_k_size = bsz * num_head * max_seq_len * dim_head; + + T *cache_k_ptr = cache_kv_data; + T *cache_v_ptr = cache_kv_data + cache_k_size; + + write_cache_kv(dev_ctx, cache_k_ptr, cache_v_ptr, k_ptr, v_ptr, bsz, + num_head, seq_len, max_seq_len, dim_head); + } else { // not generation + // TODO(wangxi): can remove dropout in inference + fmha_compute.ComputeForward( + qkv_out, cache_kv, src_mask, &transpose_out_2, cache_kv_out, + &qk_out, &src_mask_out, &softmax_out, &attn_dropout_mask_out, + &attn_dropout_out, &qktv_out, &fmha_out); + } +#ifdef _DEBUG_FUSED_MULTI_TRANSFORMER + VLOG(0) << "step3"; +#endif + + // step4. out_linear + out_linear_compute.ComputeForward(out_linear_weights[i], &fmha_out, + nullptr, buf1, nullptr); + AllReduce(*buf1, ring_id, dev_ctx); +#ifdef _DEBUG_FUSED_MULTI_TRANSFORMER + VLOG(0) << "step4"; +#endif + + // step5. ln(residual + dropout(input + bias)) + if (pre_layer_norm) { + auto *ln_scale_data = ffn_ln_scales[i]->data(); + auto *ln_bias_data = ffn_ln_biases[i]->data(); + auto *out_linear_bias_data = out_linear_biases[i]->data(); + + // inplace + fused_dropout_layernorm_helper.LayernormResidualDropoutBias( + dev_ctx, buf1->data(), x_data, out_linear_bias_data, + ln_scale_data, ln_bias_data, bias_dropout_residual_out_data, + dropout_mask_out_data, buf1->data(), ln_mean_data, ln_var_data); + } else { + } +#ifdef _DEBUG_FUSED_MULTI_TRANSFORMER + VLOG(0) << "step5"; +#endif + + // step6. ffn matmul1 + ffn1_linear_compute.ComputeForward(ffn1_weights[i], buf1, nullptr, + &ffn1_out, nullptr); +#ifdef _DEBUG_FUSED_MULTI_TRANSFORMER + VLOG(0) << "step6"; +#endif + + // step7. act bias + // TODO(wangxi): remove dropout mask in inference + fused_act_dropout_helper.DropoutActBias( + dev_ctx, ffn1_out_data, ffn1_biases[i]->data(), "gelu", + ffn1_dropout_out_data, ffn1_dropout_mask_data); +#ifdef _DEBUG_FUSED_MULTI_TRANSFORMER + VLOG(0) << "step7"; +#endif + + // step8. ffn matmul2 + ffn2_linear_compute.ComputeForward(ffn2_weights[i], &ffn1_dropout_out, + nullptr, buf1, nullptr); +#ifdef _DEBUG_FUSED_MULTI_TRANSFORMER + VLOG(0) << "step8.0"; +#endif + + AllReduce(*buf1, ring_id, dev_ctx); +#ifdef _DEBUG_FUSED_MULTI_TRANSFORMER + VLOG(0) << "step8.1"; +#endif + + // step9. residual bias + if (pre_layer_norm) { + // TODO(wangxi): remove dropout mask in inference + if (i < layers - 1) { + auto *ln_scale_data = ln_scales[i + 1]->data(); + auto *ln_bias_data = ln_biases[i + 1]->data(); + ffn2_fused_dropout_helper.LayernormResidualDropoutBias( + dev_ctx, buf1->data(), bias_dropout_residual_out_data, + ffn2_biases[i]->data(), ln_scale_data, ln_bias_data, + buf1->data(), dropout_mask_out_data, buf0->data(), + ln_mean_data, ln_var_data); + } else { + ffn2_fused_dropout_helper.ResidualDropoutBias( + dev_ctx, buf1->data(), bias_dropout_residual_out_data, + ffn2_biases[i]->data(), buf1->data(), + dropout_mask_out_data); + } + } else { + } +#ifdef _DEBUG_FUSED_MULTI_TRANSFORMER + VLOG(0) << "step9"; +#endif + x_data = buf1->data(); + std::swap(buf0, buf1); + } + } +}; + +} // namespace operators +} // namespace paddle + +namespace ops = paddle::operators; +namespace plat = paddle::platform; +REGISTER_OP_CUDA_KERNEL(fused_multi_transformer, + ops::FusedMultiTransformerOpKernel, + ops::FusedMultiTransformerOpKernel); diff --git a/paddle/fluid/operators/fused/fused_residual_dropout_bias_test.cu b/paddle/fluid/operators/fused/fused_residual_dropout_bias_test.cu index 5dff5e2225f4f..caceac1228e0a 100644 --- a/paddle/fluid/operators/fused/fused_residual_dropout_bias_test.cu +++ b/paddle/fluid/operators/fused/fused_residual_dropout_bias_test.cu @@ -147,6 +147,7 @@ struct TestFusedResidualDropoutBias { dropout_prob, is_upscale_in_train, is_test); } ctx->Wait(); + PADDLE_ENFORCE_GPU_SUCCESS(platform::GpuGetLastError()); // add residual for (int i = 0; i < rows; i++) { for (int j = 0; j < cols; j++) { @@ -186,6 +187,7 @@ struct TestFusedResidualDropoutBias { src.data(), residual.data(), bias_ptr, mask.data(), out.data(), *ctx); ctx->Wait(); + PADDLE_ENFORCE_GPU_SUCCESS(platform::GpuGetLastError()); } void FusedBackward() { @@ -313,3 +315,20 @@ TEST(FusedDropout, GPUFusedResidualDropoutBiasLargeShape) { test.CheckOut(static_cast(1e-5)); test.CheckGrad(static_cast(1e-3)); } + +TEST(FusedDropout, GPUFusedResidualDropoutBiasLargeShapeFp16) { + // Used to test that `cudaErrorLaunchOutOfResources` will not occur + int rows = 1; + int cols = 12288; + if (std::getenv("_rows") != nullptr) { + rows = atoi(std::getenv("_rows")); + } + if (std::getenv("_cols") != nullptr) { + cols = atoi(std::getenv("_cols")); + } + TestFusedResidualDropoutBias test(rows, cols, 0, 0.0, true, + true); + test.Run(); + test.CheckOut(static_cast(1e-1)); + test.CheckGrad(static_cast(1e-1)); +} diff --git a/paddle/fluid/operators/mlu/mlu_baseop.cc b/paddle/fluid/operators/mlu/mlu_baseop.cc index 793aa2644b548..eacab46800580 100644 --- a/paddle/fluid/operators/mlu/mlu_baseop.cc +++ b/paddle/fluid/operators/mlu/mlu_baseop.cc @@ -44,6 +44,32 @@ bool MLUSupportsCast(const VT::Type& src_type, const VT::Type& dst_type) { return false; } +const std::shared_ptr& GetMLURandomGenerator( + const ExecutionContext& ctx, const int64_t device_id, const int seed) { + static int64_t num_mlu_devices = -1; + static std::once_flag num_devices_init_flag; + static std::deque mlu_device_flags; + static std::vector> + mlu_rand_generators; + + std::call_once(num_devices_init_flag, []() { + num_mlu_devices = paddle::platform::GetMLUDeviceCount(); + mlu_device_flags.resize(num_mlu_devices); + mlu_rand_generators.resize(num_mlu_devices); + }); + if (device_id < 0) { + PADDLE_THROW(platform::errors::InvalidArgument( + "mlu device id shoule be greater than 0")); + } + + std::call_once(mlu_device_flags[device_id], [&]() { + mlu_rand_generators[device_id].reset( + new MLUCnnlRandomGeneratorDesc(ctx, seed)); + VLOG(4) << "device_id: " << device_id << ", initial seed: " << seed; + }); + return mlu_rand_generators[device_id]; +} + class MLUCnnlTensorDescPool { public: cnnlTensorDescriptor_t Pop() { @@ -266,23 +292,32 @@ MLUCnnlPoolingDesc::~MLUCnnlPoolingDesc() { } } -MLUCnnlRandomGeneratorDesc::MLUCnnlRandomGeneratorDesc(const bool is_mlu200, - const int seed) { - if (is_mlu200) { - PADDLE_ENFORCE_MLU_SUCCESS( - cnnlRandCreateGenerator(&mlu_generator, CNNL_RAND_RNG_FAST)); - } else { - PADDLE_ENFORCE_MLU_SUCCESS( - cnnlRandCreateGenerator(&mlu_generator, CNNL_RAND_RNG_MTGP32)); - PADDLE_ENFORCE_MLU_SUCCESS( - cnnlRandSetPseudoRandomGeneratorSeed(mlu_generator, seed)); - } +MLUCnnlRandomGeneratorDesc::MLUCnnlRandomGeneratorDesc( + const ExecutionContext& ctx, const int seed) { + PADDLE_ENFORCE_MLU_SUCCESS( + cnnlRandCreateGenerator(&mlu_generator, CNNL_RAND_RNG_MTGP32)); + PADDLE_ENFORCE_MLU_SUCCESS( + cnnlRandSetPseudoRandomGeneratorSeed(mlu_generator, seed)); + size_t workspace_size; + PADDLE_ENFORCE_MLU_SUCCESS( + cnnlRandGetMTGP32StateSize(mlu_generator, &workspace_size)); + + auto& dev_ctx = GetDevCtxFromCTX(ctx); + mlu_state = ctx.AllocateTmpTensor( + {static_cast(workspace_size)}, dev_ctx); + void* mlu_state_ptr = mlu_state.mutable_data(ctx.GetPlace()); + + cnnlHandle_t handle = GetHandleFromCTX(ctx); + PADDLE_ENFORCE_MLU_SUCCESS(cnnlRandMakeMTGP32KernelState( + handle, mlu_state_ptr, nullptr, nullptr, seed)); } const cnnlRandGenerator_t MLUCnnlRandomGeneratorDesc::get() const { return mlu_generator; } +Tensor& MLUCnnlRandomGeneratorDesc::get_state() { return mlu_state; } + MLUCnnlRandomGeneratorDesc::~MLUCnnlRandomGeneratorDesc() { if (mlu_generator) { PADDLE_ENFORCE_MLU_SUCCESS(cnnlRandDestroyGenerator(mlu_generator)); @@ -947,6 +982,26 @@ MLUCnnlTrigonDesc::~MLUCnnlTrigonDesc() { workspace_ptr, workspace_size, beta_ptr, output_desc, output)); } +/* static */ void MLUCnnl::MulAx(const ExecutionContext& ctx, + const cnnlTensorDescriptor_t alpha_desc, + const void* alpha, + const cnnlTensorDescriptor_t output_desc, + void* output) { + cnnlHandle_t handle = GetHandleFromCTX(ctx); + + size_t workspace_size; + PADDLE_ENFORCE_MLU_SUCCESS( + cnnlGetAxWorkspaceSize(handle, alpha_desc, output_desc, &workspace_size)); + + auto& dev_ctx = GetDevCtxFromCTX(ctx); + Tensor workspace = ctx.AllocateTmpTensor( + {static_cast(workspace_size)}, dev_ctx); + void* workspace_ptr = workspace.mutable_data(ctx.GetPlace()); + + PADDLE_ENFORCE_MLU_SUCCESS(cnnlAx_v2(handle, alpha_desc, alpha, output_desc, + output, workspace_ptr, workspace_size)); +} + /* static */ void MLUCnnl::BiasAddGrad( const ExecutionContext& ctx, const int axis, const cnnlTensorDescriptor_t out_backprop_desc, const void* out_backprop, @@ -959,12 +1014,23 @@ MLUCnnlTrigonDesc::~MLUCnnlTrigonDesc() { /* static */ void MLUCnnl::RandomUniform( const ExecutionContext& ctx, const int num, const cnnlDataType_t data_type, - const cnnlRandGenerator_t mlu_generator, const float min, const float max, - void* output) { + const cnnlRandGenerator_t mlu_generator, void* mlu_state, void* output) { cnnlHandle_t handle = GetHandleFromCTX(ctx); PADDLE_ENFORCE_MLU_SUCCESS(cnnlRandGenerateUniform( - handle, mlu_generator, data_type, nullptr, num, min, max, output)); + handle, mlu_generator, data_type, mlu_state, num, 0, 1, output)); +} + +/* static */ void MLUCnnl::FusedDropout( + const ExecutionContext& ctx, const cnnlRandGenerator_t generator, + const cnnlTensorDescriptor_t input_desc, const void* input, const float p, + void* state, const cnnlTensorDescriptor_t mask_desc, const void* mask, + const cnnlTensorDescriptor_t output_desc, void* output) { + cnnlHandle_t handle = GetHandleFromCTX(ctx); + + PADDLE_ENFORCE_MLU_SUCCESS(cnnlFusedDropout_v2(handle, generator, input_desc, + input, p, state, mask_desc, + mask, output_desc, output)); } /* static */ void MLUCnnl::TopK( diff --git a/paddle/fluid/operators/mlu/mlu_baseop.h b/paddle/fluid/operators/mlu/mlu_baseop.h index 9948c45e24692..572b7aa2bbd01 100644 --- a/paddle/fluid/operators/mlu/mlu_baseop.h +++ b/paddle/fluid/operators/mlu/mlu_baseop.h @@ -273,14 +273,19 @@ class MLUCnnlPoolingDesc { class MLUCnnlRandomGeneratorDesc { public: - MLUCnnlRandomGeneratorDesc(const bool is_mlu200, const int seed); + MLUCnnlRandomGeneratorDesc(const ExecutionContext& ctx, const int seed); const cnnlRandGenerator_t get() const; + Tensor& get_state(); ~MLUCnnlRandomGeneratorDesc(); private: + Tensor mlu_state; cnnlRandGenerator_t mlu_generator = nullptr; }; +const std::shared_ptr& GetMLURandomGenerator( + const ExecutionContext& ctx, const int64_t device_id, const int seed); + class MLUCnnlReduceDesc { public: MLUCnnlReduceDesc(const MLUCnnlReduceDesc& desc) = delete; @@ -537,7 +542,13 @@ class MLUCnnl { static void RandomUniform(const ExecutionContext& ctx, const int num, const cnnlDataType_t data_type, const cnnlRandGenerator_t mlu_generator, - const float min, const float max, void* output); + void* mlu_state, void* output); + + static void FusedDropout( + const ExecutionContext& ctx, const cnnlRandGenerator_t generator, + const cnnlTensorDescriptor_t input_desc, const void* input, const float p, + void* state, const cnnlTensorDescriptor_t mask_desc, const void* mask, + const cnnlTensorDescriptor_t output_desc, void* output); static void Cumsum(const ExecutionContext& ctx, const int axis, const bool exclusive, const bool reverse, @@ -709,6 +720,10 @@ class MLUCnnl { const void* in0, const cnnlTensorDescriptor_t in1_desc, const void* in1, const cnnlTensorDescriptor_t output_desc, void* output); + static void MulAx(const ExecutionContext& ctx, + const cnnlTensorDescriptor_t alpha_desc, const void* alpha, + const cnnlTensorDescriptor_t output_desc, void* output); + static void OpTensor(const ExecutionContext& ctx, const cnnlOpTensorDescriptor_t op_tensor_desc, const cnnlTensorDescriptor_t a_desc, const void* a, diff --git a/paddle/fluid/operators/optimizers/distributed_fused_lamb_op.cc b/paddle/fluid/operators/optimizers/distributed_fused_lamb_op.cc index 161483c3420fc..0159e250d317e 100644 --- a/paddle/fluid/operators/optimizers/distributed_fused_lamb_op.cc +++ b/paddle/fluid/operators/optimizers/distributed_fused_lamb_op.cc @@ -100,6 +100,10 @@ class DistributedFusedLambOpMaker : public framework::OpProtoAndCheckerMaker { .AsDispensable(); AddOutput("FP16FusedParamOut", "The updated FP16FusedParam.") .AsDispensable(); + AddOutput("FP32AccFusedGrad", "The accumulated FP32 gradients.") + .AsDispensable(); + AddOutput("FP16AccFusedGrad", "The accumulated FP16 gradients.") + .AsDispensable(); AddOutput("Moment1Out", "The updated Moment1."); AddOutput("Moment2Out", "The updated Moment2."); @@ -110,8 +114,14 @@ class DistributedFusedLambOpMaker : public framework::OpProtoAndCheckerMaker { .AsDuplicable(); AddOutput("FoundInf", "Whether there is NaN/Inf"); + AddOutput("AccStep", "The training steps.").AsDispensable(); + AddOutput("StopUpdate", + "Whether the parameter updating is stopped when the gradient " + "accumulated steps is less than Attr(acc_steps).") + .AsDispensable(); AddOutput("Step", "The global step which excludes the NaN/Inf step."); + AddAttr("acc_steps", "The gradient accumulation steps.").SetDefault(1); AddAttr("beta1", "The initial Beta1Pow value."); AddAttr("beta2", "The initial Beta2Pow value."); AddAttr("epsilon", diff --git a/paddle/fluid/operators/optimizers/distributed_fused_lamb_op.cu b/paddle/fluid/operators/optimizers/distributed_fused_lamb_op.cu index f445a140f27a3..c857c6de4d093 100644 --- a/paddle/fluid/operators/optimizers/distributed_fused_lamb_op.cu +++ b/paddle/fluid/operators/optimizers/distributed_fused_lamb_op.cu @@ -1041,6 +1041,58 @@ static void CheckHasNanInfGrad(const float *fp32_grad, int fp32_numel, } } +template +static __global__ void ElementwiseAddWithCastCUDAKernel(const T1 *x, + const T2 *y, T3 *z, + int n) { + static_assert(sizeof(T1) <= sizeof(T2), + "sizeof(T1) must be smaller than sizeof(T2)."); + using MT = MasterT; + + int i = (threadIdx.x + blockIdx.x * blockDim.x) * VecSize; + int stride = (blockDim.x * gridDim.x) * VecSize; + for (; i + VecSize <= n; i += stride) { + phi::AlignedVector x_vec; + phi::AlignedVector y_vec; + phi::AlignedVector z_vec; + phi::Load(x + i, &x_vec); + phi::Load(y + i, &y_vec); +#pragma unroll + for (int j = 0; j < VecSize; ++j) { + auto x_tmp = static_cast(x_vec[j]); + auto y_tmp = static_cast(y_vec[j]); + z_vec[j] = static_cast(x_tmp + y_tmp); + } + phi::Store(z_vec, z + i); + } + + for (; i < n; ++i) { + auto x_tmp = static_cast(x[i]); + auto y_tmp = static_cast(y[i]); + z[i] = static_cast(x_tmp + y_tmp); + } +} + +template +static void LaunchElementwiseAddWithCastKernel( + const platform::CUDADeviceContext &dev_ctx, const T1 *x, const T2 *y, T3 *z, + int n, gpuStream_t stream) { + int vec_size = + std::min(std::min(GetChunkedVecSize(x, 0), GetChunkedVecSize(y, 0)), + GetChunkedVecSize(z, 0)); + auto config = platform::GetGpuLaunchConfig1D(dev_ctx, n, vec_size); + +#define PD_LAUNCH_ELEMENTWISE_ADD_WITH_CAST_KERNEL \ + do { \ + ElementwiseAddWithCastCUDAKernel<<< \ + config.block_per_grid, config.thread_per_block, 0, stream>>>(x, y, z, \ + n); \ + } while (0) + + PD_VEC_LAUNCH_KERNEL(vec_size, PD_LAUNCH_ELEMENTWISE_ADD_WITH_CAST_KERNEL); +#undef PD_LAUNCH_ELEMENTWISE_ADD_WITH_CAST_KERNEL +} + template class DistributedFusedLambOpKernel : public framework::OpKernel { @@ -1051,6 +1103,9 @@ class DistributedFusedLambOpKernel auto stream = dev_ctx.stream(); auto place = dev_ctx.GetPlace(); + auto *found_inf_t = ctx.Output("FoundInf"); + found_inf_t->Resize({1}); + // Step 1: Get fp16 param and grad tensors int64_t fp16_numel; auto *fp16_param = GetSameInOutTensorPtr( @@ -1095,6 +1150,128 @@ class DistributedFusedLambOpKernel "Too many parameter number. Only <= %d is supported.", std::numeric_limits::max())); + auto acc_steps = ctx.Attr("acc_steps"); + PADDLE_ENFORCE_GE( + acc_steps, 1, + platform::errors::InvalidArgument( + "The gradient accumulation steps should be not less than 1.")); + if (acc_steps > 1) { + auto *step_t = ctx.Output("AccStep"); + PADDLE_ENFORCE_NOT_NULL( + step_t, + platform::errors::InvalidArgument( + "Output(AccStep) cannot be nullptr when Attr(acc_steps) > 1.")); + bool is_initialized = step_t->IsInitialized(); + int64_t *step_ptr; + if (is_initialized) { + step_ptr = step_t->mutable_data(platform::CPUPlace()); + ++(*step_ptr); + } else { + step_t->Resize({1}); + step_ptr = step_t->mutable_data(platform::CPUPlace()); + *step_ptr = 1; + } + int64_t rounded_step = (*step_ptr) % acc_steps; + + float *fp32_acc_grad = nullptr; + if (has_fp32_param) { + auto *fp32_acc_grad_t = + ctx.Output("FP32AccFusedGrad"); + PADDLE_ENFORCE_NOT_NULL( + fp32_acc_grad_t, platform::errors::InvalidArgument( + "Output(FP32AccFusedGrad) cannot be nullptr " + "when Attr(acc_steps) > 1.")); + if (!fp32_acc_grad_t->IsInitialized()) { + fp32_acc_grad_t->Resize({static_cast(fp32_numel)}); + fp32_acc_grad = fp32_acc_grad_t->mutable_data(place); + } else { + fp32_acc_grad = fp32_acc_grad_t->data(); + } + } + + platform::float16 *fp16_acc_grad = nullptr; + float *master_acc_grad = nullptr; + if (has_fp16_param) { + auto *fp16_acc_grad_t = + ctx.Output("FP16AccFusedGrad"); + PADDLE_ENFORCE_NOT_NULL( + fp16_acc_grad_t, platform::errors::InvalidArgument( + "Output(FP16AccFusedGrad) cannot be nullptr " + "when Attr(acc_steps) > 1.")); + if (!fp16_acc_grad_t->IsInitialized()) { + fp16_acc_grad_t->Resize({static_cast(3 * fp16_numel)}); + fp16_acc_grad = + fp16_acc_grad_t->mutable_data(place); + } else { + fp16_acc_grad = fp16_acc_grad_t->data(); + } + master_acc_grad = reinterpret_cast(fp16_acc_grad + fp16_numel); + } + + // Inplace addto + if (has_fp32_param) { + if (rounded_step == 1) { + memory::Copy(place, fp32_acc_grad, place, fp32_grad, + fp32_numel * sizeof(float), stream); + } else { + LaunchElementwiseAddWithCastKernel(dev_ctx, fp32_grad, fp32_acc_grad, + fp32_acc_grad, fp32_numel, stream); + } + } + + if (has_fp16_param) { + if (acc_steps == 2) { + if (rounded_step == 0) { + LaunchElementwiseAddWithCastKernel(dev_ctx, fp16_acc_grad, + fp16_grad, fp16_acc_grad, + fp16_numel, stream); + } else { + memory::Copy(place, fp16_acc_grad, place, fp16_grad, + fp16_numel * sizeof(platform::float16), stream); + } + } else { // acc_steps >= 3 + if (rounded_step == 0) { + LaunchElementwiseAddWithCastKernel(dev_ctx, fp16_grad, + master_acc_grad, fp16_acc_grad, + fp16_numel, stream); + } else if (rounded_step == 1) { + memory::Copy(place, fp16_acc_grad, place, fp16_grad, + fp16_numel * sizeof(platform::float16), stream); + } else if (rounded_step == 2) { + LaunchElementwiseAddWithCastKernel(dev_ctx, fp16_grad, + fp16_acc_grad, master_acc_grad, + fp16_numel, stream); + } else { + LaunchElementwiseAddWithCastKernel(dev_ctx, fp16_grad, + master_acc_grad, master_acc_grad, + fp16_numel, stream); + } + } + } + + auto *stop_update_t = ctx.Output("StopUpdate"); + stop_update_t->Resize({1}); + auto *stop_update = + stop_update_t->mutable_data(platform::CPUPlace()); + + auto *found_inf_cpu = + found_inf_t->mutable_data(platform::CPUPlace()); + + if (rounded_step != 0) { + *stop_update = true; + auto *found_inf_cpu = + found_inf_t->mutable_data(platform::CPUPlace()); + *found_inf_cpu = false; + return; + } else { + // swap pointer + fp32_grad = fp32_acc_grad; + fp16_grad = fp16_acc_grad; + *stop_update = false; + found_inf_t->clear(); + } + } + // Step 3: Get ParamInfo const auto *param_info_tensor = GetInputTensorPtr(ctx, "ParamInfo"); auto fp32_local_start_idx = param_info_tensor[0]; @@ -1122,7 +1299,7 @@ class DistributedFusedLambOpKernel << " , fp16_global_param_num = " << fp16_global_param_num; // Step 4: Get LearningRate, Moment1, Moment2, Beta1Pow, Beta2Pow, - // GlobalScale, FoundInf + // GlobalScale const auto *global_scale = GetInputTensorPtr(ctx, "GlobalScale"); const auto *lr = GetInputTensorPtr(ctx, "LearningRate"); int64_t partial_numel = 0; @@ -1157,8 +1334,6 @@ class DistributedFusedLambOpKernel auto *beta2pow = GetSameInOutTensorPtr(ctx, place, "Beta2Pow", "Beta2PowOut"); - auto *found_inf_t = ctx.Output("FoundInf"); - found_inf_t->Resize({1}); auto *found_inf = found_inf_t->mutable_data(place); // Step 5: Get attributes weight_decay, beta1, beta2, epsilon, diff --git a/paddle/fluid/operators/pixel_unshuffle_op.cc b/paddle/fluid/operators/pixel_unshuffle_op.cc new file mode 100644 index 0000000000000..8d16e02c04c83 --- /dev/null +++ b/paddle/fluid/operators/pixel_unshuffle_op.cc @@ -0,0 +1,103 @@ +// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "paddle/fluid/framework/infershape_utils.h" +#include "paddle/fluid/framework/op_registry.h" +#include "paddle/fluid/framework/op_version_registry.h" +#include "paddle/phi/core/infermeta_utils.h" +#include "paddle/phi/infermeta/backward.h" +#include "paddle/phi/infermeta/unary.h" + +namespace paddle { +namespace operators { + +class PixelUnshuffleOp : public framework::OperatorWithKernel { + public: + using framework::OperatorWithKernel::OperatorWithKernel; +}; + +class PixelUnshuffleOpMaker : public framework::OpProtoAndCheckerMaker { + public: + void Make() override { + AddInput("X", + "(Tensor, default Tensor), " + "the input feature data of PixelUnshuffleOp, the layout is " + "[N, C, H, W] or [N, H, W, C]."); + AddOutput("Out", + "(Tensor, default Tensor), the output of " + "PixelUnshuffleOp. The layout is [N, C*factor^2, H/factor, " + "W/factor] or [N, H/factor, W/factor, C*factor^2]."); + AddAttr("downscale_factor", + "the factor to decrease spatial resolution by.") + .SetDefault(1); + AddAttr( + "data_format", + "An optional string from: \"NHWC\", \"NCHW\". " + "Defaults to \"NHWC\", Specify the data format of the input data.") + .SetDefault("NCHW"); + + AddComment(R"DOC( + Pixel Unshuffle operator + This operator rearranges elements in a tensor of shape :math:`(*, C, H, W)` + to a tensor of shape :math:`(*, C\times r^2, H / r, W / r)`. + + This operation is the reversion of PixelShuffle operation. + + Please refer to the paper: + `Real-Time Single Image and Video Super-Resolution Using an Efficient + Sub-Pixel Convolutional Neural Network `_ + by Shi et. al (2016) for more details. + + )DOC"); + } +}; + +template +class PixelUnshuffleGradOpMaker : public framework::SingleGradOpMaker { + public: + using framework::SingleGradOpMaker::SingleGradOpMaker; + + protected: + void Apply(GradOpPtr op) const override { + op->SetType("pixel_unshuffle_grad"); + op->SetInput(framework::GradVarName("Out"), this->OutputGrad("Out")); + op->SetAttrMap(this->Attrs()); + op->SetOutput(framework::GradVarName("X"), this->InputGrad("X")); + } +}; + +class PixelUnshuffleGradOp : public framework::OperatorWithKernel { + public: + using framework::OperatorWithKernel::OperatorWithKernel; +}; + +} // namespace operators +} // namespace paddle + +namespace ops = paddle::operators; +DECLARE_INFER_SHAPE_FUNCTOR(pixel_unshuffle, PixelUnshuffleInferShapeFunctor, + PD_INFER_META(phi::PixelUnshuffleInferMeta)); + +REGISTER_OPERATOR(pixel_unshuffle, ops::PixelUnshuffleOp, + ops::PixelUnshuffleOpMaker, + ops::PixelUnshuffleGradOpMaker, + ops::PixelUnshuffleGradOpMaker, + PixelUnshuffleInferShapeFunctor); + +DECLARE_INFER_SHAPE_FUNCTOR(pixel_unshuffle_grad, + PixelUnshuffleGradInferShapeFunctor, + PD_INFER_META(phi::PixelUnshuffleGradInferMeta)); + +REGISTER_OPERATOR(pixel_unshuffle_grad, ops::PixelUnshuffleGradOp, + PixelUnshuffleGradInferShapeFunctor); diff --git a/paddle/fluid/operators/pscore/heter_cloud_comm_cpu_test.cc b/paddle/fluid/operators/pscore/heter_cloud_comm_cpu_test.cc index 2340f443c49fb..cf6369eecdf9c 100644 --- a/paddle/fluid/operators/pscore/heter_cloud_comm_cpu_test.cc +++ b/paddle/fluid/operators/pscore/heter_cloud_comm_cpu_test.cc @@ -15,6 +15,9 @@ limitations under the License. */ #if defined PADDLE_WITH_PSCORE #include +#include +#include +#include #include #include #include @@ -69,44 +72,6 @@ void StartSwitchServer( std::vector peer_endpoints) { switch_server_ptr->SetPeerEndPoints(peer_endpoints); switch_server_ptr->SetEndPoint(endpoints[0]); - /* - std::shared_ptr b_req_handler; - b_req_handler.reset(new distributed::SendAndRecvVariableHandler()); - switch_server_ptr->SetServiceHandler(b_req_handler); - - switch_server_ptr->SetLocalScope(); - - switch_server_ptr->RegisterServiceHandler( - std::to_string(distributed::PS_SAVE_WITH_SCOPE), - [&](const MultiVarMsg* request, MultiVarMsg* response, - brpc::Controller* cntl) -> int { - return b_req_handler->SaveInSwitchWithScope(request, response, cntl); - }); - - switch_server_ptr->RegisterServiceHandler(std::to_string(distributed::PS_SAVE_WITH_SHARD), - [&](const MultiVarMsg* request, MultiVarMsg* - response, - brpc::Controller* cntl) -> int { - return b_req_handler->SaveInSwitchWithShard( - request, response, cntl); - }); - - switch_server_ptr->RegisterServiceHandler(std::to_string(distributed::PS_QUERY_WITH_SCOPE), - [&](const MultiVarMsg* request, MultiVarMsg* - response, - brpc::Controller* cntl) -> int { - return b_req_handler->QueryInSwitchWithScope( - request, response, cntl); - }); - - switch_server_ptr->RegisterServiceHandler(std::to_string(distributed::PS_QUERY_WITH_SHARD), - [&](const MultiVarMsg* request, MultiVarMsg* - response, - brpc::Controller* cntl) -> int { - return b_req_handler->QueryInSwitchWithShard( - request, response, cntl); - }); - */ switch_server_ptr->StartHeterService(false); } @@ -119,6 +84,129 @@ void StartSwitchInterServer( switch_server_ptr->StartHeterInterService(false); } +void TestShardSendRecv( + std::shared_ptr heter_client_ptr_) { + auto send_async = [&]() -> void { + std::vector vars_len{2 * sizeof(float), + 4 * sizeof(float)}; // 字节数 + std::vector values{1.0, 2.0, 3.0, 4.0, 5.0, 6.0}; + int64_t data_size = 6 * sizeof(float); + std::vector send_var_names{"w", "x"}; + int group_id = 0; + int ret = heter_client_ptr_->Send(group_id, send_var_names, vars_len, + values.data(), data_size); + if (!ret) { + LOG(INFO) << ">>>> TestShardSendRecv: worker send success"; + } + }; + std::thread t(send_async); + + int group_id = 0; + std::vector recv_var_names{"w", "x"}; + int data_size = 6 * sizeof(float); + float* value_ptr = new float[6]; + int ret = + heter_client_ptr_->Recv(group_id, recv_var_names, value_ptr, data_size); + if (!ret) { + VLOG(4) << "queried data is: "; + for (int i = 0; i < 6; i++) { + VLOG(4) << value_ptr[i] << " "; + } + delete[] value_ptr; + LOG(INFO) << "<<<< TestShardSendRecv: worker recv success"; + } + + t.join(); +} + +void PressTestSendRecv( + std::shared_ptr heter_client_ptr_) { + // long l = 0, m = 0; + std::ifstream file("/send_20_34", std::ios::in | std::ios::binary); + // l = file.tellg(); + // file.seekg(0, std::ios::end); + // m = file.tellg(); + // file.close(); + // VLOG(0) << "size of file " << "20_34" << " is " << (m - l) << " bytes.\n"; + int64_t vars_len = 2359296 * sizeof(float); + int64_t data_size = vars_len * sizeof(float); + VLOG(0) << "float num: " << data_size; + float* data_ptr = new float[data_size]; + file.read((char*)data_ptr, 9437184); + VLOG(0) << "send data is: " << data_ptr[0] << ", " << data_ptr[1]; + std::vector var_names{"34"}; + int loopCnt = 600; + auto send_async = [&]() -> void { + int i = 0; + while (i++ < loopCnt) { + heter_client_ptr_->Send(20, var_names, {vars_len}, data_ptr, data_size); + } + }; + std::thread t(send_async); + float* values = new float[2359296]; + int i = 0; + while (i++ < loopCnt) { + int ret = heter_client_ptr_->Recv(20, var_names, values, data_size); + if (!ret) { + VLOG(0) << "diff: " << abs(values[0] - 0.159544) << ", " + << abs(values[1] + 2.3484); + VLOG(0) << "loop id: " << i; + for (int j = 0; j < 2359296; j++) { + if (abs(values[j] - data_ptr[j]) > 4e-6) { + VLOG(0) << "error data idx: " << j; + VLOG(0) << "diff detail: " << values[j] << ", " << data_ptr[j]; + LOG(INFO) << ">>>> worker recv ERROR"; + break; + } + } + for (uint32_t i = 0; i < 2359296; i++) { + values[i] = -1; // reset + } + } + } + delete[] values; + + std::ofstream recv("/recv_20_34", std::ios::out | std::ios::binary); + recv.write((char*)values, data_size); + recv.close(); + t.join(); +} + +void TestScopeSendRecv( + std::shared_ptr heter_client_ptr_) { + platform::CPUPlace place; + platform::CPUDeviceContext ctx(place); + framework::Executor exe(place); + std::shared_ptr send_scope_ptr = + std::make_shared(); + int64_t rows_numel = 10; + InitTensorsOnClient(send_scope_ptr.get(), &place, rows_numel); + LOG(INFO) << "InitTensorsOnClient done"; + auto send_async = [&]() -> void { + std::string message_name = std::to_string(distributed::PS_SAVE_WITH_SCOPE); + std::vector send_var_names{"w", "x"}; + int ret = heter_client_ptr_->Send(ctx, *send_scope_ptr, message_name, + send_var_names); + if (!ret) { + LOG(ERROR) << ">>>> TestScopeSendRecv: worker send success"; + } + }; + std::thread t(send_async); + + std::string message_name = std::to_string(distributed::PS_QUERY_WITH_SCOPE); + std::vector recv_var_names{"w", "x"}; + std::shared_ptr recv_scope_ptr = + std::make_shared(); + int ret = heter_client_ptr_->Recv(ctx, *recv_scope_ptr, message_name, + recv_var_names); + if (!ret && recv_scope_ptr->FindVar("w") && recv_scope_ptr->FindVar("x")) { + LOG(INFO) << "<<<< TestScopeSendRecv: worker recv success"; + } else { + LOG(INFO) << "<<<< TestScopeSendRecv: worker recv failed"; + } + t.join(); +} + TEST(HETERSENDANDRECV, CPU) { setenv("http_proxy", "", 1); setenv("https_proxy", "", 1); @@ -155,79 +243,19 @@ TEST(HETERSENDANDRECV, CPU) { switch_server_ptr_b->WaitServerReady(); // 获取 client 实例 + // 开启单测时，请重新设置 HeterClient 端的 recv_switch_channels_ std::shared_ptr heter_client_ptr_ = distributed::HeterClient::GetInstance( {switch_a_endpoint, switch_b_endpoint}, {}, 0); + framework::ProgramDesc program; platform::CPUPlace place; - platform::CPUDeviceContext ctx(place); framework::Executor exe(place); - - framework::ProgramDesc program; exe.Prepare(program, 0); // solve undefined symbol: tensor_table.cc - std::shared_ptr send_scope_ptr = - std::make_shared(); - int64_t rows_numel = 10; - InitTensorsOnClient(send_scope_ptr.get(), &place, rows_numel); - LOG(INFO) << "InitTensorsOnClient done"; - - auto send_async = [&]() -> void { - /* - //std::string message_name = - std::to_string(distributed::PS_SAVE_WITH_SCOPE); - std::string message_name = "send and save"; - std::vector send_var_names{"w", "x"}; - int ret = heter_client_ptr_->Send(ctx, *send_scope_ptr, message_name, - send_var_names); - if (!ret) { - LOG(ERROR) << ">>>> worker send success"; - } - */ - ///* - std::vector vars_len{2, 4}; - std::vector values{1.0, 2.0, 3.0, 4.0, 5.0, 6.0}; - int64_t data_size = 6; - std::vector send_var_names{"w", "x"}; - int group_id = 0; - int ret = heter_client_ptr_->Send(group_id, send_var_names, vars_len, - values.data(), data_size); - if (!ret) { - LOG(INFO) << ">>>> worker send success"; - } - //*/ - }; - std::thread send_thread(send_async); - /* - std::string message_name = std::to_string(distributed::PS_QUERY_WITH_SCOPE); - std::vector recv_var_names{"w", "x"}; - std::shared_ptr recv_scope_ptr = - std::make_shared(); - int ret = heter_client_ptr_->Recv(ctx, *recv_scope_ptr, message_name, - recv_var_names); - if (!ret && recv_scope_ptr->FindVar("w") && recv_scope_ptr->FindVar("x")) { - LOG(INFO) << ">>>> worker recv success"; - } else { - LOG(INFO) << "worker recv failed"; - } - */ - ///* - int group_id = 0; - std::vector recv_var_names{"w", "x"}; - std::vector values; - int data_size = 6; - values.resize(data_size); - int ret = heter_client_ptr_->Recv(group_id, recv_var_names, values.data(), - data_size); - if (!ret) { - VLOG(4) << "queried data is: "; - for (auto f : values) { - VLOG(4) << f << " "; - } - LOG(INFO) << ">>>> worker recv success"; - } - //*/ - send_thread.join(); + // TestScopeSendRecv(heter_client_ptr_); + TestShardSendRecv(heter_client_ptr_); + // PressTestSendRecv(heter_client_ptr_); switch_server_ptr_a->Stop(); LOG(INFO) << "switch server A stopped"; diff --git a/paddle/fluid/operators/reduce_ops/reduce_max_op_xpu.cc b/paddle/fluid/operators/reduce_ops/reduce_max_op_xpu.cc index 15d672da04bec..1c1269a08dbdc 100644 --- a/paddle/fluid/operators/reduce_ops/reduce_max_op_xpu.cc +++ b/paddle/fluid/operators/reduce_ops/reduce_max_op_xpu.cc @@ -105,11 +105,10 @@ class ReduceMaxGradXPUKernel : public framework::OpKernel { " wrong value[%d %s].", r, XPUAPIErrorMsg[r])); // step 2. comparse out_brocast and x - r = xpu::elementwise_equal(dev_ctx.x_context(), x_data, brocast1, equal, - x->numel()); + r = xpu::equal(dev_ctx.x_context(), x_data, brocast1, equal, x->numel()); PADDLE_ENFORCE_EQ( r == xpu::Error_t::SUCCESS, true, - platform::errors::External("XPU elementwise_equal in reduce_max_grad " + platform::errors::External("XPU equal in reduce_max_grad " "op return wrong value[%d %s].", r, XPUAPIErrorMsg[r])); // step 3. get x_grad diff --git a/paddle/fluid/operators/reduce_ops/reduce_sum_op.cc b/paddle/fluid/operators/reduce_ops/reduce_sum_op.cc index 2a78774f3706e..6b8e6b8f8054f 100644 --- a/paddle/fluid/operators/reduce_ops/reduce_sum_op.cc +++ b/paddle/fluid/operators/reduce_ops/reduce_sum_op.cc @@ -89,6 +89,12 @@ class ReduceSumVarTypeInference : public paddle::framework::VarTypeInference { BOOST_GET_CONST(int, ctx->GetAttr("out_dtype"))); if (data_type >= 0) { ctx->SetOutputDataType("Out", data_type); + } else { + auto x_type = ctx->GetInputDataType("X"); + if (x_type == framework::proto::VarType::BOOL || + x_type == framework::proto::VarType::INT32) { + ctx->SetOutputDataType("Out", framework::proto::VarType::INT64); + } } } }; diff --git a/paddle/fluid/operators/size_op.cc b/paddle/fluid/operators/size_op.cc index 84b0f403be038..4af355bfca641 100644 --- a/paddle/fluid/operators/size_op.cc +++ b/paddle/fluid/operators/size_op.cc @@ -23,6 +23,19 @@ namespace operators { class SizeOp : public framework::OperatorWithKernel { public: using framework::OperatorWithKernel::OperatorWithKernel; + + protected: + framework::OpKernelType GetExpectedKernelType( + const framework::ExecutionContext& ctx) const override { + auto dtype = framework::proto::VarType::FP32; // dtype is not important + return framework::OpKernelType(dtype, ctx.GetPlace()); + } + + framework::OpKernelType GetKernelTypeForVar( + const std::string& var_name, const framework::Tensor& tensor, + const framework::OpKernelType& expected_kernel_type) const override { + return expected_kernel_type; + } }; class SizeOpMaker : public framework::OpProtoAndCheckerMaker { @@ -40,6 +53,8 @@ Return the number of elements in the input. } }; +DECLARE_NO_NEED_BUFFER_VARS_INFERER(SizeOpNoNeedBufferVarInferer, "Input"); + } // namespace operators } // namespace paddle @@ -50,4 +65,4 @@ REGISTER_OPERATOR( size, ops::SizeOp, ops::SizeOpMaker, paddle::framework::EmptyGradOpMaker, paddle::framework::EmptyGradOpMaker, - SizeInferShapeFunctor); + SizeInferShapeFunctor, ops::SizeOpNoNeedBufferVarInferer); diff --git a/paddle/fluid/operators/transpose_op.cc b/paddle/fluid/operators/transpose_op.cc index 1a297e7238ccd..a45d32b34b983 100644 --- a/paddle/fluid/operators/transpose_op.cc +++ b/paddle/fluid/operators/transpose_op.cc @@ -90,7 +90,7 @@ class TransposeOp : public framework::OperatorWithKernel { framework::OpKernelType GetExpectedKernelType( const framework::ExecutionContext &ctx) const override { framework::LibraryType library_{framework::LibraryType::kPlain}; - std::string data_format = ctx.Attr("data_format"); + auto &data_format = ctx.Attr("data_format"); framework::DataLayout layout_ = framework::StringToDataLayout(data_format); auto data_type = OperatorWithKernel::IndicateVarDataType(ctx, "X"); #ifdef PADDLE_WITH_MKLDNN diff --git a/paddle/fluid/platform/device/xpu/xpu2_op_list.h b/paddle/fluid/platform/device/xpu/xpu2_op_list.h index 357644b62d3ed..583014b6f4773 100644 --- a/paddle/fluid/platform/device/xpu/xpu2_op_list.h +++ b/paddle/fluid/platform/device/xpu/xpu2_op_list.h @@ -57,6 +57,9 @@ XPUOpMap& get_kl2_ops() { pOpKernelType(vartype::BOOL, XPUPlace()), pOpKernelType(vartype::INT64, XPUPlace()), pOpKernelType(vartype::INT32, XPUPlace())})}, + {"check_finite_and_unscale", + XPUKernelSet({pOpKernelType(vartype::FP32, XPUPlace()), + pOpKernelType(vartype::FP16, XPUPlace())})}, {"clip", XPUKernelSet({pOpKernelType(vartype::FP32, XPUPlace())})}, {"concat_grad", XPUKernelSet({pOpKernelType(vartype::FP32, XPUPlace()), pOpKernelType(vartype::FP16, XPUPlace())})}, diff --git a/paddle/fluid/platform/dynload/cublasLt.h b/paddle/fluid/platform/dynload/cublasLt.h index c9a59751a320a..5157cfdad2e59 100644 --- a/paddle/fluid/platform/dynload/cublasLt.h +++ b/paddle/fluid/platform/dynload/cublasLt.h @@ -1,4 +1,5 @@ /* Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved. +Copyright (c) 2022 NVIDIA Authors. All Rights Reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -38,19 +39,25 @@ namespace dynload { // APIs available after CUDA 10.1 // #if CUDA_VERSION >= 10100 -#define CUBLASLT_BLAS_ROUTINE_EACH(__macro) \ - __macro(cublasLtCreate); \ - __macro(cublasLtDestroy); \ - __macro(cublasLtMatmul); \ - __macro(cublasLtMatmulDescCreate); \ - __macro(cublasLtMatmulDescDestroy); \ - __macro(cublasLtMatmulDescSetAttribute); \ - __macro(cublasLtMatrixLayoutCreate); \ - __macro(cublasLtMatrixLayoutDestroy); \ - __macro(cublasLtMatrixLayoutSetAttribute); \ - __macro(cublasLtMatrixTransform); \ - __macro(cublasLtMatrixTransformDescCreate); \ - __macro(cublasLtMatrixTransformDescDestroy); \ +#define CUBLASLT_BLAS_ROUTINE_EACH(__macro) \ + __macro(cublasLtCreate); \ + __macro(cublasLtDestroy); \ + __macro(cublasLtMatmul); \ + __macro(cublasLtMatmulDescCreate); \ + __macro(cublasLtMatmulDescDestroy); \ + __macro(cublasLtMatmulDescSetAttribute); \ + __macro(cublasLtMatmulDescGetAttribute); \ + __macro(cublasLtMatrixLayoutCreate); \ + __macro(cublasLtMatrixLayoutDestroy); \ + __macro(cublasLtMatrixLayoutSetAttribute); \ + __macro(cublasLtMatrixLayoutGetAttribute); \ + __macro(cublasLtMatmulPreferenceCreate); \ + __macro(cublasLtMatmulPreferenceDestroy); \ + __macro(cublasLtMatmulPreferenceSetAttribute); \ + __macro(cublasLtMatmulAlgoGetHeuristic); \ + __macro(cublasLtMatrixTransform); \ + __macro(cublasLtMatrixTransformDescCreate); \ + __macro(cublasLtMatrixTransformDescDestroy); \ __macro(cublasLtMatrixTransformDescSetAttribute); CUBLASLT_BLAS_ROUTINE_EACH(PLATFORM_DECLARE_DYNAMIC_LOAD_CUBLASLT_WRAP) diff --git a/paddle/fluid/platform/flags.cc b/paddle/fluid/platform/flags.cc index f89452853b49b..054a804e6b38e 100644 --- a/paddle/fluid/platform/flags.cc +++ b/paddle/fluid/platform/flags.cc @@ -1,4 +1,5 @@ // Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved. +// Copyright (c) 2022 NVIDIA Authors. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -107,6 +108,29 @@ PADDLE_DEFINE_EXPORTED_string( "share-memory only."); #endif +#if defined(PADDLE_WITH_CUDA) +/** + * CUDA related FLAG + * Name: FLAGS_cublaslt_exhaustive_search_times + * Since Version: 2.3.0 + * Value Range: int64_t, default=0 + * Example: + * Note: Represents times of exhaustive search to evaluate performance of + * cuBlasLt matmul algorithm (with/without epilogue). Set this flag + * with value > 0 to enable exhaustive search. Default is 0, means + * getting algorithms via heuristic search. There are two search methods + * in cuBlasLt, heuristic search and exhaustive search. Exhaustive search + * attempts all cuBlasLt algorithms to select the fastest, which is very + * time-consuming, and the selected algorithm will be cached for a given + * layer specification Once you change the layer specifications + * (such as M, N and K), it will re-search again. + */ +PADDLE_DEFINE_EXPORTED_int64( + cublaslt_exhaustive_search_times, 0, + "The times of exhaustive search for cuBlasLt matmul with/without " + " epilogue algorithms, default is 0, means disabling exhaustive search."); +#endif + #if defined(PADDLE_WITH_ASCEND_CL) PADDLE_DEFINE_EXPORTED_string( selected_npus, "", diff --git a/paddle/fluid/platform/profiler.cc b/paddle/fluid/platform/profiler.cc index 8fa48ffcfb158..75abf36e676d0 100644 --- a/paddle/fluid/platform/profiler.cc +++ b/paddle/fluid/platform/profiler.cc @@ -612,7 +612,7 @@ static std::map DockHostEventRecorderHostPart() { auto host_evt_sec = HostEventRecorder::GetInstance().GatherEvents(); EmulateEventPushAndPop(host_evt_sec, &thr_events); EmulateCPURecordsAdd(host_evt_sec); - return std::move(thr_events); + return thr_events; } static void DockHostEventRecorderDevicePart( diff --git a/paddle/fluid/pybind/CMakeLists.txt b/paddle/fluid/pybind/CMakeLists.txt index e616da3ab2e4c..fe7f86a9af81c 100755 --- a/paddle/fluid/pybind/CMakeLists.txt +++ b/paddle/fluid/pybind/CMakeLists.txt @@ -7,6 +7,9 @@ set(PYBIND_DEPS init pybind python proto_desc memory executor fleet_wrapper box_ if (WITH_PSCORE) set(PYBIND_DEPS ${PYBIND_DEPS} ps_service) set(PYBIND_DEPS ${PYBIND_DEPS} graph_py_service) + if (WITH_HETERPS) + set(PYBIND_DEPS ${PYBIND_DEPS} graph_gpu_wrapper) + endif() endif() if (WITH_GPU OR WITH_ROCM) set(PYBIND_DEPS ${PYBIND_DEPS} dynload_cuda) @@ -84,10 +87,6 @@ set(PYBIND_SRCS communication.cc cuda_streams_py.cc) -if (WITH_ONNXRUNTIME) - set(PYBIND_DEPS ${PYBIND_DEPS} onnxruntime_predictor) -endif() - if(NOT ON_INFER) set (PYBIND_DEPS ${PYBIND_DEPS} processgroup eager_reducer) if (WITH_NCCL) @@ -170,10 +169,6 @@ if(WITH_PYTHON) list(APPEND OP_FUNCTION_GENERETOR_DEPS hccl_context) endif(WITH_ASCEND_CL) - if (WITH_ONNXRUNTIME) - list(APPEND OP_FUNCTION_GENERETOR_DEPS onnxruntime_predictor) - endif() - if(WITH_CNCL) list(APPEND OP_FUNCTION_GENERETOR_DEPS cncl_context) endif(WITH_CNCL) diff --git a/paddle/fluid/pybind/bind_fleet_executor.h b/paddle/fluid/pybind/bind_fleet_executor.h index 733701fa36ba8..f9568819688e5 100644 --- a/paddle/fluid/pybind/bind_fleet_executor.h +++ b/paddle/fluid/pybind/bind_fleet_executor.h @@ -14,6 +14,10 @@ #pragma once +#if defined(_MSC_VER) +#include +typedef SSIZE_T ssize_t; +#endif #include namespace paddle { diff --git a/paddle/fluid/pybind/compatible.h b/paddle/fluid/pybind/compatible.h index f9d4cf5888fee..5f7628e5f2ab9 100644 --- a/paddle/fluid/pybind/compatible.h +++ b/paddle/fluid/pybind/compatible.h @@ -14,6 +14,10 @@ #pragma once +#if defined(_MSC_VER) +#include +typedef SSIZE_T ssize_t; +#endif #include namespace paddle { diff --git a/paddle/fluid/pybind/eager.cc b/paddle/fluid/pybind/eager.cc index 8695928205bb0..6601c8e8e3e4d 100644 --- a/paddle/fluid/pybind/eager.cc +++ b/paddle/fluid/pybind/eager.cc @@ -146,10 +146,13 @@ void InitTensorWithNumpyValue(TensorObject* self, const py::object& array, zero_copy); } else if (platform::is_npu_place(place)) { SetTensorFromPyArray(impl_ptr, array, place, zero_copy); + } else if (platform::is_custom_place(place)) { + SetTensorFromPyArray(impl_ptr, array, place, + zero_copy); } else { PADDLE_THROW(platform::errors::InvalidArgument( "Place should be one of " - "CPUPlace/XPUPlace/CUDAPlace/CUDAPinnedPlace/NPUPlace")); + "CPUPlace/XPUPlace/CUDAPlace/CUDAPinnedPlace/NPUPlace/CustomPlace")); } } diff --git a/paddle/fluid/pybind/eager_functions.cc b/paddle/fluid/pybind/eager_functions.cc index 1073cdc83a428..4d7b50943d084 100644 --- a/paddle/fluid/pybind/eager_functions.cc +++ b/paddle/fluid/pybind/eager_functions.cc @@ -9,6 +9,12 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ // disable numpy compile error + +#if defined(_MSC_VER) +#include +typedef SSIZE_T ssize_t; +#endif + #include #include diff --git a/paddle/fluid/pybind/eager_method.cc b/paddle/fluid/pybind/eager_method.cc index 13fba2baa1d6c..e6bd1c0b52682 100644 --- a/paddle/fluid/pybind/eager_method.cc +++ b/paddle/fluid/pybind/eager_method.cc @@ -9,6 +9,12 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ // disable numpy compile error + +#if defined(_MSC_VER) +#include +typedef SSIZE_T ssize_t; +#endif + #include #include diff --git a/paddle/fluid/pybind/eager_utils.cc b/paddle/fluid/pybind/eager_utils.cc index 9719963d51da0..d07cbd5ee21a2 100644 --- a/paddle/fluid/pybind/eager_utils.cc +++ b/paddle/fluid/pybind/eager_utils.cc @@ -46,6 +46,7 @@ extern PyTypeObject* g_cpuplace_pytype; extern PyTypeObject* g_xpuplace_pytype; extern PyTypeObject* g_npuplace_pytype; extern PyTypeObject* g_cudapinnedplace_pytype; +extern PyTypeObject* g_customplace_pytype; extern PyTypeObject* g_framework_tensor_pytype; extern PyTypeObject* g_framework_lodtensorarray_pytype; extern PyTypeObject* g_custom_op_kernel_ctx_pytype; @@ -377,10 +378,15 @@ platform::Place CastPyArg2Place(PyObject* obj, ssize_t arg_pos) { } else if (PyObject_IsInstance( obj, reinterpret_cast(g_cudapinnedplace_pytype))) { place = ::pybind11::handle(obj).cast(); + } else if (PyObject_IsInstance( + obj, reinterpret_cast(g_customplace_pytype))) { + place = ::pybind11::handle(obj).cast(); } else { PADDLE_THROW(platform::errors::InvalidArgument( "argument (position %d) must be " - "one of(Place,CUDAPlace,CPUPlace,XPUPlace,NPUPlace,CUDAPinnedPlace), " + "one " + "of(Place,CUDAPlace,CPUPlace,XPUPlace,NPUPlace,CUDAPinnedPlace," + "CustomPlace), " "but got %s", arg_pos + 1, reinterpret_cast(obj->ob_type)->tp_name)); } @@ -1019,7 +1025,20 @@ paddle::experimental::Scalar CastNumpy2Scalar(PyObject* obj, PyTypeObject* type = obj->ob_type; auto type_name = std::string(type->tp_name); VLOG(1) << "type_name: " << type_name; - if (type_name == "numpy.float64") { + if (type_name == "numpy.ndarray" && PySequence_Check(obj)) { + PyObject* item = nullptr; + item = PySequence_GetItem(obj, 0); + if (PyObject_CheckFloatOrToFloat(&item)) { + float value = static_cast(PyFloat_AsDouble(item)); + return paddle::experimental::Scalar(value); + } else { + PADDLE_THROW(platform::errors::InvalidArgument( + "%s(): argument (position %d) is numpy.ndarry, the inner elements " + "must be " + "numpy.float32/float64 now, but got %s", + op_type, arg_pos + 1, type_name)); // NOLINT + } + } else if (type_name == "numpy.float64") { double value = CastPyArg2Double(obj, op_type, arg_pos); return paddle::experimental::Scalar(value); } else if (type_name == "numpy.float32") { @@ -1058,7 +1077,7 @@ paddle::experimental::Scalar CastPyArg2Scalar(PyObject* obj, bool value = CastPyArg2Boolean(obj, op_type, arg_pos); return paddle::experimental::Scalar(value); } else if (PyLong_Check(obj)) { - int value = CastPyArg2Int(obj, op_type, arg_pos); + int64_t value = CastPyArg2Long(obj, op_type, arg_pos); return paddle::experimental::Scalar(value); } else if (PyFloat_Check(obj)) { float value = CastPyArg2Float(obj, op_type, arg_pos); diff --git a/paddle/fluid/pybind/eager_utils.h b/paddle/fluid/pybind/eager_utils.h index 22c41073c9dd7..c4ddb34763228 100644 --- a/paddle/fluid/pybind/eager_utils.h +++ b/paddle/fluid/pybind/eager_utils.h @@ -10,6 +10,11 @@ See the License for the specific language governing permissions and limitations under the License. */ #pragma once +#if defined(_MSC_VER) +#include +typedef SSIZE_T ssize_t; +#endif + #include #include "paddle/phi/common/backend.h" #include "paddle/phi/common/data_type.h" diff --git a/paddle/fluid/pybind/fleet_py.cc b/paddle/fluid/pybind/fleet_py.cc index 8d8301689521b..4df43dc1a3a52 100644 --- a/paddle/fluid/pybind/fleet_py.cc +++ b/paddle/fluid/pybind/fleet_py.cc @@ -1,11 +1,8 @@ /* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved. - Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at - http://www.apache.org/licenses/LICENSE-2.0 - Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -37,6 +34,7 @@ limitations under the License. */ #include "paddle/fluid/distributed/ps/service/heter_client.h" #include "paddle/fluid/distributed/ps/service/ps_service/graph_py_service.h" #include "paddle/fluid/distributed/ps/wrapper/fleet.h" +#include "paddle/fluid/framework/fleet/heter_ps/graph_gpu_wrapper.h" namespace py = pybind11; using paddle::distributed::CommContext; @@ -59,11 +57,7 @@ void BindDistFleetWrapper(py::module* m) { .def("load_model", &FleetWrapper::LoadModel) .def("load_one_table", &FleetWrapper::LoadModelOneTable) .def("init_server", &FleetWrapper::InitServer) - .def("run_server", - (uint64_t (FleetWrapper::*)(void)) & FleetWrapper::RunServer) - .def("run_server", (uint64_t (FleetWrapper::*)( // NOLINT - const std::string&, uint32_t)) & // NOLINT - FleetWrapper::RunServer) + .def("run_server", &FleetWrapper::RunServer) .def("init_worker", &FleetWrapper::InitWorker) .def("push_dense_params", &FleetWrapper::PushDenseParamSync) .def("pull_dense_params", &FleetWrapper::PullDenseVarsSync) @@ -78,7 +72,11 @@ void BindDistFleetWrapper(py::module* m) { .def("set_clients", &FleetWrapper::SetClients) .def("get_client_info", &FleetWrapper::GetClientsInfo) .def("create_client2client_connection", - &FleetWrapper::CreateClient2ClientConnection); + &FleetWrapper::CreateClient2ClientConnection) + .def("client_flush", &FleetWrapper::ClientFlush) + .def("get_cache_threshold", &FleetWrapper::GetCacheThreshold) + .def("cache_shuffle", &FleetWrapper::CacheShuffle) + .def("save_cache", &FleetWrapper::SaveCache); } void BindPSHost(py::module* m) { @@ -212,8 +210,8 @@ void BindGraphPyClient(py::module* m) { .def("start_client", &GraphPyClient::start_client) .def("batch_sample_neighboors", &GraphPyClient::batch_sample_neighbors) .def("batch_sample_neighbors", &GraphPyClient::batch_sample_neighbors) - .def("use_neighbors_sample_cache", - &GraphPyClient::use_neighbors_sample_cache) + // .def("use_neighbors_sample_cache", + // &GraphPyClient::use_neighbors_sample_cache) .def("remove_graph_node", &GraphPyClient::remove_graph_node) .def("random_sample_nodes", &GraphPyClient::random_sample_nodes) .def("stop_server", &GraphPyClient::StopServer) @@ -251,6 +249,12 @@ void BindGraphPyClient(py::module* m) { using paddle::distributed::TreeIndex; using paddle::distributed::IndexWrapper; using paddle::distributed::IndexNode; +#ifdef PADDLE_WITH_HETERPS +using paddle::framework::GraphGpuWrapper; +using paddle::framework::NeighborSampleResult; +using paddle::framework::NeighborSampleQuery; +using paddle::framework::NodeQueryResult; +#endif void BindIndexNode(py::module* m) { py::class_(*m, "IndexNode") @@ -301,6 +305,47 @@ void BindIndexWrapper(py::module* m) { .def("clear_tree", &IndexWrapper::clear_tree); } +#ifdef PADDLE_WITH_HETERPS +void BindNodeQueryResult(py::module* m) { + py::class_(*m, "NodeQueryResult") + .def(py::init<>()) + .def("initialize", &NodeQueryResult::initialize) + .def("display", &NodeQueryResult::display) + .def("get_val", &NodeQueryResult::get_val) + .def("get_len", &NodeQueryResult::get_len); +} +void BindNeighborSampleQuery(py::module* m) { + py::class_(*m, "NeighborSampleQuery") + .def(py::init<>()) + .def("initialize", &NeighborSampleQuery::initialize) + .def("display", &NeighborSampleQuery::display); +} + +void BindNeighborSampleResult(py::module* m) { + py::class_(*m, "NeighborSampleResult") + .def(py::init<>()) + .def("initialize", &NeighborSampleResult::initialize) + .def("display", &NeighborSampleResult::display); +} + +void BindGraphGpuWrapper(py::module* m) { + py::class_(*m, "GraphGpuWrapper") + .def(py::init<>()) + //.def("test", &GraphGpuWrapper::test) + .def("initialize", &GraphGpuWrapper::initialize) + .def("neighbor_sample", &GraphGpuWrapper::graph_neighbor_sample_v3) + .def("graph_neighbor_sample", &GraphGpuWrapper::graph_neighbor_sample) + .def("set_device", &GraphGpuWrapper::set_device) + .def("init_service", &GraphGpuWrapper::init_service) + .def("set_up_types", &GraphGpuWrapper::set_up_types) + .def("query_node_list", &GraphGpuWrapper::query_node_list) + .def("add_table_feat_conf", &GraphGpuWrapper::add_table_feat_conf) + .def("load_edge_file", &GraphGpuWrapper::load_edge_file) + .def("upload_batch", &GraphGpuWrapper::upload_batch) + .def("load_node_file", &GraphGpuWrapper::load_node_file); +} +#endif + using paddle::distributed::IndexSampler; using paddle::distributed::LayerWiseSampler; diff --git a/paddle/fluid/pybind/fleet_py.h b/paddle/fluid/pybind/fleet_py.h index 206a69f5a8019..a47aec749bda5 100644 --- a/paddle/fluid/pybind/fleet_py.h +++ b/paddle/fluid/pybind/fleet_py.h @@ -36,5 +36,11 @@ void BindIndexNode(py::module* m); void BindTreeIndex(py::module* m); void BindIndexWrapper(py::module* m); void BindIndexSampler(py::module* m); +#ifdef PADDLE_WITH_HETERPS +void BindNeighborSampleResult(py::module* m); +void BindGraphGpuWrapper(py::module* m); +void BindNodeQueryResult(py::module* m); +void BindNeighborSampleQuery(py::module* m); +#endif } // namespace pybind } // namespace paddle diff --git a/paddle/fluid/pybind/imperative.cc b/paddle/fluid/pybind/imperative.cc index a610206dd9cd1..fd94d74cc6bbc 100644 --- a/paddle/fluid/pybind/imperative.cc +++ b/paddle/fluid/pybind/imperative.cc @@ -2029,35 +2029,35 @@ void BindImperative(py::module *m_ptr) { *(imperative::AmpOperators::Instance().GetMutableAllowOps()), *(imperative::AmpOperators::Instance().GetMutableBlockOps())); }) - .def( - "_get_kernel_signature", - [](imperative::Tracer &self, const std::string &type, - const PyNameVarBaseMap &ins, const PyNameVarBaseMap &outs, - framework::AttributeMap attrs) { - // TODO(xiongkun): move this function outside of tracer. - auto ins_map = ConvertToNameTensorMap(ins); - auto outs_map = ConvertToNameTensorMap(outs); - { - auto input_to_vector = - [](paddle::SmallVector &vec) { - return std::vector(vec.begin(), vec.end()); - }; - auto output_to_vector = - [](paddle::SmallVector &vec) { - return std::vector(vec.begin(), vec.end()); - }; - auto attr_to_vector = [](paddle::SmallVector &vec) { - return std::vector(vec.begin(), vec.end()); - }; - auto ret = self.GetExpectedKernelSignature(type, ins_map, - outs_map, attrs); - auto kernelsig_ins = input_to_vector(std::get<0>(ret.args)); - auto kernelsig_attrs = attr_to_vector(std::get<1>(ret.args)); - auto kernelsig_outs = output_to_vector(std::get<2>(ret.args)); - return std::make_tuple(kernelsig_ins, kernelsig_attrs, - kernelsig_outs); - } - }) + .def("_get_kernel_signature", + [](imperative::Tracer &self, const std::string &type, + const PyNameVarBaseMap &ins, const PyNameVarBaseMap &outs, + framework::AttributeMap attrs) { + // TODO(xiongkun): move this function outside of tracer. + auto ins_map = ConvertToNameTensorMap(ins); + auto outs_map = ConvertToNameTensorMap(outs); + { + auto input_to_vector = + [](paddle::small_vector &vec) { + return std::vector(vec.begin(), vec.end()); + }; + auto output_to_vector = + [](paddle::small_vector &vec) { + return std::vector(vec.begin(), vec.end()); + }; + auto attr_to_vector = + [](paddle::small_vector &vec) { + return std::vector(vec.begin(), vec.end()); + }; + auto ret = self.GetExpectedKernelSignature(type, ins_map, + outs_map, attrs); + auto kernelsig_ins = input_to_vector(ret.input_names); + auto kernelsig_attrs = attr_to_vector(ret.attr_names); + auto kernelsig_outs = output_to_vector(ret.output_names); + return std::make_tuple(kernelsig_ins, kernelsig_attrs, + kernelsig_outs); + } + }) .def("trace", [](imperative::Tracer &self, const std::string &type, const PyNameVarBaseMap &ins, const PyNameVarBaseMap &outs, diff --git a/paddle/fluid/pybind/inference_api.cc b/paddle/fluid/pybind/inference_api.cc index 91d5d39622714..1bbe6808b2846 100644 --- a/paddle/fluid/pybind/inference_api.cc +++ b/paddle/fluid/pybind/inference_api.cc @@ -765,10 +765,7 @@ void BindMkldnnQuantizerConfig(py::module *m) { return; }) .def("set_quant_batch_size", &MkldnnQuantizerConfig::SetWarmupBatchSize) - .def( - "set_enabled_op_types", - (void (MkldnnQuantizerConfig::*)(std::unordered_set &)) & - MkldnnQuantizerConfig::SetEnabledOpTypes); + .def("set_enabled_op_types", &MkldnnQuantizerConfig::SetEnabledOpTypes); } #endif diff --git a/paddle/fluid/pybind/inference_api.h b/paddle/fluid/pybind/inference_api.h index c2adfbecf72ca..300d3b480e113 100644 --- a/paddle/fluid/pybind/inference_api.h +++ b/paddle/fluid/pybind/inference_api.h @@ -14,6 +14,11 @@ #pragma once +#if defined(_MSC_VER) +#include +typedef SSIZE_T ssize_t; +#endif + #include namespace paddle { diff --git a/paddle/fluid/pybind/io.h b/paddle/fluid/pybind/io.h index dfe3154cb95da..942c93deccf99 100644 --- a/paddle/fluid/pybind/io.h +++ b/paddle/fluid/pybind/io.h @@ -14,6 +14,11 @@ limitations under the License. */ #pragma once +#if defined(_MSC_VER) +#include +typedef SSIZE_T ssize_t; +#endif + #include #include "paddle/fluid/pybind/pybind_boost_headers.h" diff --git a/paddle/fluid/pybind/kernel_signature_generator.cc b/paddle/fluid/pybind/kernel_signature_generator.cc index 1520174fba288..0b0a8628b14f1 100644 --- a/paddle/fluid/pybind/kernel_signature_generator.cc +++ b/paddle/fluid/pybind/kernel_signature_generator.cc @@ -58,10 +58,10 @@ int main(int argc, char **argv) { if (kernel_signature_map.Has(op_name)) { kernel_signature_map_str = kernel_signature_map_str + "\"" + op_kernel_pair.first + "\":{"; - auto &args = kernel_signature_map.Get(op_name).args; + const auto &args = kernel_signature_map.Get(op_name); kernel_signature_map_str += "\"inputs\":["; - auto inputs_ = std::get<0>(args); + auto inputs_ = args.input_names; for (size_t i = 0; i < inputs_.size(); i++) { kernel_signature_map_str = kernel_signature_map_str + "\"" + inputs_[i] + "\","; @@ -69,14 +69,14 @@ int main(int argc, char **argv) { if (inputs_.size()) kernel_signature_map_str.pop_back(); kernel_signature_map_str += "],\"attrs\":["; - auto attrs_ = std::get<1>(args); + auto attrs_ = args.attr_names; for (size_t i = 0; i < attrs_.size(); i++) { kernel_signature_map_str = kernel_signature_map_str + "\"" + attrs_[i] + "\","; } if (attrs_.size()) kernel_signature_map_str.pop_back(); kernel_signature_map_str += "],\"outputs\":["; - auto outputs_ = std::get<2>(args); + auto outputs_ = args.output_names; for (size_t i = 0; i < outputs_.size(); i++) { kernel_signature_map_str = kernel_signature_map_str + "\"" + outputs_[i] + "\","; diff --git a/paddle/fluid/pybind/op_function.h b/paddle/fluid/pybind/op_function.h index 7b9379df6be2c..5a5650e75665c 100644 --- a/paddle/fluid/pybind/op_function.h +++ b/paddle/fluid/pybind/op_function.h @@ -177,7 +177,7 @@ static inline void HandleViewBetweenInputAndOutput( } } -PyObject* MakeReturnPyObject( +static inline PyObject* MakeReturnPyObject( const std::shared_ptr& out) { return ::pybind11::detail::type_caster_base::cast_holder( ::pybind11::detail::holder_helper< @@ -186,7 +186,7 @@ PyObject* MakeReturnPyObject( .ptr(); } -PyObject* MakeReturnPyObject( +static inline PyObject* MakeReturnPyObject( const std::vector>& out) { PyObject* result = PyList_New((Py_ssize_t)out.size()); diff --git a/paddle/fluid/pybind/op_function_common.cc b/paddle/fluid/pybind/op_function_common.cc index 50e0daf8508e3..5eed63d0800b3 100644 --- a/paddle/fluid/pybind/op_function_common.cc +++ b/paddle/fluid/pybind/op_function_common.cc @@ -153,7 +153,7 @@ void CastPyArg2AttrInt(PyObject* obj, int64_t CastPyArg2Long(PyObject* obj, const std::string& op_type, ssize_t arg_pos) { if (PyObject_CheckLongOrToLong(&obj)) { - return (int64_t)PyLong_AsLong(obj); // NOLINT + return (int64_t)PyLong_AsLongLong(obj); // NOLINT } else { PADDLE_THROW(platform::errors::InvalidArgument( "%s(): argument (position %d) must be " diff --git a/paddle/fluid/pybind/op_function_common.h b/paddle/fluid/pybind/op_function_common.h index debaf8fae17b7..549da39d9b891 100644 --- a/paddle/fluid/pybind/op_function_common.h +++ b/paddle/fluid/pybind/op_function_common.h @@ -14,6 +14,11 @@ #pragma once +#if defined(_MSC_VER) +#include +typedef SSIZE_T ssize_t; +#endif + #include #include #include diff --git a/paddle/fluid/pybind/op_function_generator.h b/paddle/fluid/pybind/op_function_generator.h index 7b128bd3b0e4d..2b849968c76f9 100644 --- a/paddle/fluid/pybind/op_function_generator.h +++ b/paddle/fluid/pybind/op_function_generator.h @@ -32,6 +32,10 @@ std::map> op_ins_map = { {"fused_attention", {"X", "LnScale", "LnBias", "QKVW", "QKVBias", "CacheKV", "SrcMask", "OutLinearW", "OutLinearBias", "Ln2Scale", "Ln2Bias"}}, + {"fused_multi_transformer", + {"X", "LnScale", "LnBias", "QKVW", "QKVBias", "CacheKV", "TimeStep", + "SrcMask", "OutLinearW", "OutLinearBias", "FFNLnScale", "FFNLnBias", + "FFN1Weight", "FFN1Bias", "FFN2Weight", "FFN2Bias"}}, {"instance_norm", {"X", "Scale", "Bias"}}, {"gru_unit", {"Input", "HiddenPrev", "Weight", "Bias"}}, {"label_smooth", {"X", "PriorDist"}}, @@ -176,6 +180,7 @@ std::map> op_outs_map = { {"lamb", {"ParamOut", "Moment1Out", "Moment2Out", "Beta1PowOut", "Beta2PowOut", "MasterParamOut"}}, + {"fused_multi_transformer", {"CacheKVOut", "Out"}}, }; // NOTE(zhiqiu): Commonly, the outputs in auto-generated OP function are @@ -253,6 +258,7 @@ std::map> op_passing_outs_map = { {"assign_value", {"Out"}}, {"split", {"Out"}}, {"concat", {"Out"}}, + {"fused_multi_transformer", {"CacheKVOut"}}, }; // NOTE(pangyoki): Tensor View Strategy. diff --git a/paddle/fluid/pybind/protobuf.h b/paddle/fluid/pybind/protobuf.h index 4c5aa9701cd5a..54b788cccba5b 100644 --- a/paddle/fluid/pybind/protobuf.h +++ b/paddle/fluid/pybind/protobuf.h @@ -13,6 +13,10 @@ See the License for the specific language governing permissions and limitations under the License. */ #pragma once +#if defined(_MSC_VER) +#include +typedef SSIZE_T ssize_t; +#endif #include #include diff --git a/paddle/fluid/pybind/pybind.cc b/paddle/fluid/pybind/pybind.cc index b135af43ab174..3a242fe2582a5 100644 --- a/paddle/fluid/pybind/pybind.cc +++ b/paddle/fluid/pybind/pybind.cc @@ -193,6 +193,7 @@ PyTypeObject *g_xpuplace_pytype = nullptr; PyTypeObject *g_npuplace_pytype = nullptr; PyTypeObject *g_cudapinnedplace_pytype = nullptr; PyTypeObject *g_mluplace_pytype = nullptr; +PyTypeObject *g_customplace_pytype = nullptr; PyTypeObject *g_framework_tensor_pytype = nullptr; PyTypeObject *g_framework_lodtensorarray_pytype = nullptr; PyTypeObject *g_custom_op_kernel_ctx_pytype = nullptr; @@ -1920,7 +1921,7 @@ All parameter, weight, gradient are variables in Paddle. Prune the backward part of a program, mostly called in program.clone(for_test=True). - Args: + Args: program (ProgramDesc): The original program. Returns: @@ -1929,6 +1930,17 @@ All parameter, weight, gradient are variables in Paddle. which contains the id pair of pruned block and corresponding origin block. )DOC"); + m.def("get_readable_comile_key", [](const OpDesc &op_desc) { + auto compilation_key = + BOOST_GET_CONST(std::string, op_desc.GetAttr("compilation_key")); + VLOG(4) << std::hash{}(compilation_key) << " " + << compilation_key.size(); + proto::ProgramDesc desc; + desc.ParseFromString(compilation_key); + auto s = desc.DebugString(); + VLOG(4) << s; + return s; + }); m.def("empty_var_name", []() { return std::string(framework::kEmptyVarName); }); m.def("grad_var_suffix", @@ -2125,8 +2137,8 @@ All parameter, weight, gradient are variables in Paddle. #endif return devices; }); - py::class_(m, "CustomPlace", - R"DOC( + py::class_ customplace(m, "CustomPlace", + R"DOC( CustomPlace is a descriptor of a device. It represents a custom device on which a tensor will be allocated and a model will run. @@ -2135,7 +2147,9 @@ All parameter, weight, gradient are variables in Paddle. import paddle fake_cpu_place = paddle.CustomPlace("FakeCPU", 0) - )DOC") + )DOC"); + g_customplace_pytype = reinterpret_cast(customplace.ptr()); + customplace .def("__init__", [](platform::CustomPlace &self, const std::string &device_type, int dev_id) { @@ -2192,6 +2206,7 @@ All parameter, weight, gradient are variables in Paddle. std::exit(-1); #endif }) + .def("_type", &PlaceIndex) .def("get_device_id", [](const platform::CustomPlace &self) { return self.GetDeviceId(); }) .def("get_device_type", @@ -4563,6 +4578,12 @@ All parameter, weight, gradient are variables in Paddle. BindTreeIndex(&m); BindIndexWrapper(&m); BindIndexSampler(&m); +#ifdef PADDLE_WITH_HETERPS + BindNodeQueryResult(&m); + BindNeighborSampleQuery(&m); + BindNeighborSampleResult(&m); + BindGraphGpuWrapper(&m); +#endif #endif } } // namespace pybind diff --git a/paddle/fluid/pybind/pybind_boost_headers.h b/paddle/fluid/pybind/pybind_boost_headers.h index 3eb4db175a745..be9333eb7361b 100644 --- a/paddle/fluid/pybind/pybind_boost_headers.h +++ b/paddle/fluid/pybind/pybind_boost_headers.h @@ -45,10 +45,28 @@ struct PYBIND11_HIDDEN paddle_variant_caster_visitor paddle_variant_caster_visitor(return_value_policy policy, handle parent) : policy(policy), parent(parent) {} - template - handle operator()(T const &src) const { + template ::value, + bool>::type* = nullptr> + handle operator()(T const& src) const { return make_caster::cast(src, policy, parent); } + + template ::value, + bool>::type* = nullptr> + handle operator()(T const& src) const { + try { + return make_caster::cast(src, policy, parent); + } catch (std::exception& ex) { + VLOG(4) << ex.what(); + VLOG(4) << src; + // UnicodeDecodeError, src is not utf-8 encoded + // see details: + // https://github.com/pybind/pybind11/blob/master/docs/advanced/cast/strings.rst + return PYBIND11_BYTES_FROM_STRING_AND_SIZE(src.data(), src.size()); + } + } }; template @@ -105,7 +123,7 @@ struct paddle_variant_caster> { return load_success_; } - static handle cast(Type const &src, return_value_policy policy, + static handle cast(Type const& src, return_value_policy policy, handle parent) { paddle_variant_caster_visitor visitor(policy, parent); return boost::apply_visitor(visitor, src); diff --git a/paddle/infrt/dialect/phi/pass/kernel_op_desc.cc b/paddle/infrt/dialect/phi/pass/kernel_op_desc.cc index b1aa81260968f..9425a290142da 100644 --- a/paddle/infrt/dialect/phi/pass/kernel_op_desc.cc +++ b/paddle/infrt/dialect/phi/pass/kernel_op_desc.cc @@ -93,9 +93,9 @@ std::vector GetCandidateKernels( phi_kernel_desc.input_types.clear(); phi_kernel_desc.output_types.clear(); phi::KernelArgsDef args_def = kernel_key_map.at(kernel_key).args_def(); - const paddle::SmallVector& + const paddle::small_vector& input_arg = args_def.input_defs(); - const paddle::SmallVector& + const paddle::small_vector& output_arg = args_def.output_defs(); for (auto tensor_arg : input_arg) { phi_kernel_desc.input_types.emplace_back(ConvertPlaceFromPhi(tensor_arg)); diff --git a/paddle/infrt/dialect/phi/pass/phi_op_convert_pass.cc b/paddle/infrt/dialect/phi/pass/phi_op_convert_pass.cc index 4bf39d4f66094..862c9ae4ee5af 100644 --- a/paddle/infrt/dialect/phi/pass/phi_op_convert_pass.cc +++ b/paddle/infrt/dialect/phi/pass/phi_op_convert_pass.cc @@ -193,14 +193,14 @@ void PhiOpConvertPass::convertStage() { op->replaceAllUsesWith(kernel_op.getResults()); } else { ::phi::KernelSignature kernel_sign = - ::phi::OpUtilsMap::Instance().GetArgumentMappingFn(op_name)( + (*::phi::OpUtilsMap::Instance().GetArgumentMappingFn(op_name))( infrt::ProtoArgumentMappingContext(op)); VLOG(3) << "IncompatiblePhiKernel: op(" << op_name << "), kernel(" << kernel_sign.name << ")"; // resort input&output according to kernel_sign ::llvm::SmallVector inputs, ori_output; ::llvm::SmallVector output_types; - for (const std::string &str : std::get<0>(kernel_sign.args)) { + for (const std::string &str : kernel_sign.input_names) { if (pd_dialect_inputs_info_map_.at(op_name).count(str) == 0) { LOG(ERROR) << "No input info for Op " << op_name << " and argument " << str; @@ -210,7 +210,7 @@ void PhiOpConvertPass::convertStage() { inputs.push_back(op->getOperands()[index]); } - for (const std::string &str : std::get<2>(kernel_sign.args)) { + for (const std::string &str : kernel_sign.output_names) { if (pd_dialect_outputs_info_map_.at(op_name).count(str) == 0) { LOG(ERROR) << "No output info for Op " << op_name << " and argument " << str; diff --git a/paddle/infrt/dialect/phi/pass/proto_arg_map_context.cc b/paddle/infrt/dialect/phi/pass/proto_arg_map_context.cc index 070867853ad3e..49fe069217ed7 100644 --- a/paddle/infrt/dialect/phi/pass/proto_arg_map_context.cc +++ b/paddle/infrt/dialect/phi/pass/proto_arg_map_context.cc @@ -63,6 +63,12 @@ bool ProtoArgumentMappingContext::IsDenseTensorInput( const std::string& name) const { return true; } + +bool ProtoArgumentMappingContext::IsDenseTensorInputs( + const std::string& name) const { + return true; +} + bool ProtoArgumentMappingContext::IsSelectedRowsInput( const std::string& name) const { return false; diff --git a/paddle/infrt/dialect/phi/pass/proto_arg_map_context.h b/paddle/infrt/dialect/phi/pass/proto_arg_map_context.h index 5cf2ef979076d..7cb2651ccf6a2 100644 --- a/paddle/infrt/dialect/phi/pass/proto_arg_map_context.h +++ b/paddle/infrt/dialect/phi/pass/proto_arg_map_context.h @@ -41,6 +41,7 @@ class ProtoArgumentMappingContext : public ::phi::ArgumentMappingContext { size_t OutputSize(const std::string& name) const override; bool IsDenseTensorInput(const std::string& name) const override; + bool IsDenseTensorInputs(const std::string& name) const override; bool IsSelectedRowsInput(const std::string& name) const override; bool IsDenseTensorVectorInput(const std::string& name) const override; diff --git a/paddle/phi/CMakeLists.txt b/paddle/phi/CMakeLists.txt index d43e327393f25..0595ea4d8bddf 100644 --- a/paddle/phi/CMakeLists.txt +++ b/paddle/phi/CMakeLists.txt @@ -27,7 +27,7 @@ set(PHI_DEPS convert_utils dense_tensor phi_context kernel_factory kernel_contex get_property(phi_kernels GLOBAL PROPERTY PHI_KERNELS) set(PHI_DEPS ${PHI_DEPS} ${phi_kernels}) -cc_library(phi DEPS ${PHI_DEPS}) +create_dummy_static_lib(phi LIBS ${PHI_DEPS} LIMIT 100) set(phi_extension_header_file ${CMAKE_CURRENT_SOURCE_DIR}/extension.h CACHE INTERNAL "phi/extension.h file") file(WRITE ${phi_extension_header_file} "// Header file generated by paddle/phi/CMakeLists.txt for external users,\n// DO NOT edit or include it within paddle.\n\n#pragma once\n\n") diff --git a/paddle/phi/api/lib/api_custom_impl.cc b/paddle/phi/api/lib/api_custom_impl.cc index ae248a7bf1280..38a60ab978900 100644 --- a/paddle/phi/api/lib/api_custom_impl.cc +++ b/paddle/phi/api/lib/api_custom_impl.cc @@ -69,7 +69,12 @@ std::tuple adam_impl( kernel_data_type = kernel_key.dtype(); } } + std::string kernel_name = "adam"; + if (!phi::DenseTensor::classof(grad.impl().get())) { + kernel_name = "adam_dense_param_sparse_grad"; + } + const auto& kernel = phi::KernelFactory::Instance().SelectKernelOrThrowError( kernel_name, {kernel_backend, kernel_layout, kernel_data_type}); VLOG(6) << kernel_name << " API kernel key: [" << kernel_backend << ", " @@ -77,9 +82,7 @@ std::tuple adam_impl( VLOG(6) << kernel_name << " API kernel: " << kernel; auto* dev_ctx = GetDeviceContextByBackend(kernel_backend); - auto input_param = PrepareData(param, kernel.InputAt(0), {}); - auto input_grad = PrepareData(grad, kernel.InputAt(1), {}); auto input_lr = PrepareData(learning_rate, kernel.InputAt(2), {}); auto input_moment1 = PrepareData(moment1, kernel.InputAt(3), {}); auto input_moment2 = PrepareData(moment2, kernel.InputAt(4), {}); @@ -140,78 +143,155 @@ std::tuple adam_impl( phi::MetaTensor meta_out_4(kernel_out_4); phi::MetaTensor meta_out_5(kernel_out_5); - phi::AdamInferMeta(MakeMetaTensor(*input_param), - MakeMetaTensor(*input_grad), - MakeMetaTensor(*input_lr), - MakeMetaTensor(*input_moment1), - MakeMetaTensor(*input_moment2), - MakeMetaTensor(*input_beta1_pow), - MakeMetaTensor(*input_beta2_pow), - input_meta_ref_master_param, - input_meta_ref_skip_update, - beta1, - beta2, - epsilon, - lazy_mode, - min_row_size_to_use_multithread, - multi_precision, - use_global_beta_pow, - &meta_out_0, - &meta_out_1, - &meta_out_2, - &meta_out_3, - &meta_out_4, - &meta_out_5); - - using kernel_signature = void (*)(const platform::DeviceContext&, - const phi::DenseTensor&, - const phi::DenseTensor&, - const phi::DenseTensor&, - const phi::DenseTensor&, - const phi::DenseTensor&, - const phi::DenseTensor&, - const phi::DenseTensor&, - paddle::optional, - paddle::optional, - const Scalar&, - const Scalar&, - const Scalar&, - bool, - int64_t, - bool, - bool, - phi::DenseTensor*, - phi::DenseTensor*, - phi::DenseTensor*, - phi::DenseTensor*, - phi::DenseTensor*, - phi::DenseTensor*); - auto* kernel_fn = kernel.GetVariadicKernelFn(); + if (phi::DenseTensor::classof(grad.impl().get())) { + auto input_grad = PrepareData(grad, kernel.InputAt(1), {}); + + phi::AdamInferMeta(MakeMetaTensor(*input_param), + MakeMetaTensor(*input_grad), + MakeMetaTensor(*input_lr), + MakeMetaTensor(*input_moment1), + MakeMetaTensor(*input_moment2), + MakeMetaTensor(*input_beta1_pow), + MakeMetaTensor(*input_beta2_pow), + input_meta_ref_master_param, + input_meta_ref_skip_update, + beta1, + beta2, + epsilon, + lazy_mode, + min_row_size_to_use_multithread, + multi_precision, + use_global_beta_pow, + &meta_out_0, + &meta_out_1, + &meta_out_2, + &meta_out_3, + &meta_out_4, + &meta_out_5); + + using kernel_signature = void (*)(const platform::DeviceContext&, + const phi::DenseTensor&, + const phi::DenseTensor&, + const phi::DenseTensor&, + const phi::DenseTensor&, + const phi::DenseTensor&, + const phi::DenseTensor&, + const phi::DenseTensor&, + paddle::optional, + paddle::optional, + const Scalar&, + const Scalar&, + const Scalar&, + bool, + int64_t, + bool, + bool, + phi::DenseTensor*, + phi::DenseTensor*, + phi::DenseTensor*, + phi::DenseTensor*, + phi::DenseTensor*, + phi::DenseTensor*); + auto* kernel_fn = kernel.GetVariadicKernelFn(); - (*kernel_fn)(*dev_ctx, - *input_param, - *input_grad, - *input_lr, - *input_moment1, - *input_moment2, - *input_beta1_pow, - *input_beta2_pow, - input_master_param, - input_skip_update, - beta1, - beta2, - epsilon, - lazy_mode, - min_row_size_to_use_multithread, - multi_precision, - use_global_beta_pow, - kernel_out_0, - kernel_out_1, - kernel_out_2, - kernel_out_3, - kernel_out_4, - kernel_out_5); + (*kernel_fn)(*dev_ctx, + *input_param, + *input_grad, + *input_lr, + *input_moment1, + *input_moment2, + *input_beta1_pow, + *input_beta2_pow, + input_master_param, + input_skip_update, + beta1, + beta2, + epsilon, + lazy_mode, + min_row_size_to_use_multithread, + multi_precision, + use_global_beta_pow, + kernel_out_0, + kernel_out_1, + kernel_out_2, + kernel_out_3, + kernel_out_4, + kernel_out_5); + } else { + auto input_grad = TensorToSelectedRows(grad); + + phi::AdamInferMeta(MakeMetaTensor(*input_param), + MakeMetaTensor(*input_grad), + MakeMetaTensor(*input_lr), + MakeMetaTensor(*input_moment1), + MakeMetaTensor(*input_moment2), + MakeMetaTensor(*input_beta1_pow), + MakeMetaTensor(*input_beta2_pow), + input_meta_ref_master_param, + input_meta_ref_skip_update, + beta1, + beta2, + epsilon, + lazy_mode, + min_row_size_to_use_multithread, + multi_precision, + use_global_beta_pow, + &meta_out_0, + &meta_out_1, + &meta_out_2, + &meta_out_3, + &meta_out_4, + &meta_out_5); + + using kernel_signature = void (*)(const platform::DeviceContext&, + const phi::DenseTensor&, + const phi::SelectedRows&, + const phi::DenseTensor&, + const phi::DenseTensor&, + const phi::DenseTensor&, + const phi::DenseTensor&, + const phi::DenseTensor&, + paddle::optional, + paddle::optional, + const Scalar&, + const Scalar&, + const Scalar&, + bool, + int64_t, + bool, + bool, + phi::DenseTensor*, + phi::DenseTensor*, + phi::DenseTensor*, + phi::DenseTensor*, + phi::DenseTensor*, + phi::DenseTensor*); + auto* kernel_fn = kernel.GetVariadicKernelFn(); + (*kernel_fn)(*dev_ctx, + *input_param, + *input_grad, + *input_lr, + *input_moment1, + *input_moment2, + *input_beta1_pow, + *input_beta2_pow, + input_master_param, + input_skip_update, + beta1, + beta2, + epsilon, + lazy_mode, + min_row_size_to_use_multithread, + multi_precision, + use_global_beta_pow, + kernel_out_0, + kernel_out_1, + kernel_out_2, + kernel_out_3, + kernel_out_4, + kernel_out_5); + } return api_output; } diff --git a/paddle/phi/api/lib/api_gen_utils.cc b/paddle/phi/api/lib/api_gen_utils.cc index e0c910ba3d66c..fb205212ff371 100644 --- a/paddle/phi/api/lib/api_gen_utils.cc +++ b/paddle/phi/api/lib/api_gen_utils.cc @@ -41,7 +41,7 @@ std::unique_ptr> TensorToDenseTensor( *std::dynamic_pointer_cast(t.impl())); } - return std::move(pt_tensors); + return pt_tensors; } std::shared_ptr TensorToSelectedRows(const Tensor& tensor) { @@ -154,7 +154,7 @@ phi::TensorBase* SetSparseKernelOutput(Tensor* out, TensorType type) { std::make_shared(phi::DenseTensor(), phi::DenseTensor(), phi::DenseTensor(), - phi::DDim{-1}); + phi::DDim{-1, -1}); out->set_impl(sparse_tensor); return sparse_tensor.get(); } else { diff --git a/paddle/phi/api/lib/data_transform.cc b/paddle/phi/api/lib/data_transform.cc index 65cb37d414299..58827a98503ce 100644 --- a/paddle/phi/api/lib/data_transform.cc +++ b/paddle/phi/api/lib/data_transform.cc @@ -253,7 +253,7 @@ std::unique_ptr> PrepareData( } } - return std::move(pt_tensors); + return pt_tensors; } } // namespace experimental diff --git a/paddle/phi/api/lib/tensor.cc b/paddle/phi/api/lib/tensor.cc index be0a937c91e4f..a7b89d7a4dca9 100644 --- a/paddle/phi/api/lib/tensor.cc +++ b/paddle/phi/api/lib/tensor.cc @@ -341,7 +341,11 @@ bool Tensor::is_initialized() const { return defined() && impl_->initialized(); } -void Tensor::reset() { impl_.reset(); } +void Tensor::reset() { + impl_.reset(); + autograd_meta_.reset(); + name_ = ""; +} /* Part 6: Operator overloading */ diff --git a/paddle/phi/backends/dynload/cublasLt.h b/paddle/phi/backends/dynload/cublasLt.h index a1562370c377b..4c7ac9c3f21c4 100644 --- a/paddle/phi/backends/dynload/cublasLt.h +++ b/paddle/phi/backends/dynload/cublasLt.h @@ -1,4 +1,5 @@ /* Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved. +Copyright (c) 2022 NVIDIA Authors. All Rights Reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -52,19 +53,25 @@ extern void *cublasLt_dso_handle; // APIs available after CUDA 10.1 // #if CUDA_VERSION >= 10100 -#define CUBLASLT_BLAS_ROUTINE_EACH(__macro) \ - __macro(cublasLtCreate); \ - __macro(cublasLtDestroy); \ - __macro(cublasLtMatmul); \ - __macro(cublasLtMatmulDescCreate); \ - __macro(cublasLtMatmulDescDestroy); \ - __macro(cublasLtMatmulDescSetAttribute); \ - __macro(cublasLtMatrixLayoutCreate); \ - __macro(cublasLtMatrixLayoutDestroy); \ - __macro(cublasLtMatrixLayoutSetAttribute); \ - __macro(cublasLtMatrixTransform); \ - __macro(cublasLtMatrixTransformDescCreate); \ - __macro(cublasLtMatrixTransformDescDestroy); \ +#define CUBLASLT_BLAS_ROUTINE_EACH(__macro) \ + __macro(cublasLtCreate); \ + __macro(cublasLtDestroy); \ + __macro(cublasLtMatmul); \ + __macro(cublasLtMatmulDescCreate); \ + __macro(cublasLtMatmulDescDestroy); \ + __macro(cublasLtMatmulDescSetAttribute); \ + __macro(cublasLtMatmulDescGetAttribute); \ + __macro(cublasLtMatrixLayoutCreate); \ + __macro(cublasLtMatrixLayoutDestroy); \ + __macro(cublasLtMatrixLayoutSetAttribute); \ + __macro(cublasLtMatrixLayoutGetAttribute); \ + __macro(cublasLtMatmulPreferenceCreate); \ + __macro(cublasLtMatmulPreferenceDestroy); \ + __macro(cublasLtMatmulPreferenceSetAttribute); \ + __macro(cublasLtMatmulAlgoGetHeuristic); \ + __macro(cublasLtMatrixTransform); \ + __macro(cublasLtMatrixTransformDescCreate); \ + __macro(cublasLtMatrixTransformDescDestroy); \ __macro(cublasLtMatrixTransformDescSetAttribute); CUBLASLT_BLAS_ROUTINE_EACH(DECLARE_DYNAMIC_LOAD_CUBLASLT_WRAP) diff --git a/paddle/phi/core/attribute.h b/paddle/phi/core/attribute.h new file mode 100644 index 0000000000000..d1b2920335576 --- /dev/null +++ b/paddle/phi/core/attribute.h @@ -0,0 +1,50 @@ +// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#include +#include + +#include "paddle/phi/common/data_type.h" +#include "paddle/phi/common/int_array.h" +#include "paddle/phi/common/layout.h" +#include "paddle/phi/common/scalar.h" +#include "paddle/utils/variant.h" + +namespace phi { + +class Place; + +// NOTE: Add needed type in the future +using Attribute = paddle::variant, + std::vector, + std::vector, + std::vector, + std::vector, + std::vector, + Scalar, + std::vector, + IntArray, + DataType, + DataLayout, + Place>; + +} // namespace phi diff --git a/paddle/phi/core/compat/arg_map_context.cc b/paddle/phi/core/compat/arg_map_context.cc index 6f678966badd9..800245406afd3 100644 --- a/paddle/phi/core/compat/arg_map_context.cc +++ b/paddle/phi/core/compat/arg_map_context.cc @@ -20,11 +20,11 @@ limitations under the License. */ namespace phi { std::ostream& operator<<(std::ostream& os, KernelSignature signature) { os << "Kernel Signature - name: " << signature.name << "; inputs: " - << paddle::string::join_strings(std::get<0>(signature.args), ", ") + << paddle::string::join_strings(signature.input_names, ", ") << "; attributes: " - << paddle::string::join_strings(std::get<1>(signature.args), ", ") + << paddle::string::join_strings(signature.attr_names, ", ") << "; outputs: " - << paddle::string::join_strings(std::get<2>(signature.args), ", "); + << paddle::string::join_strings(signature.output_names, ", "); return os; } diff --git a/paddle/phi/core/compat/arg_map_context.h b/paddle/phi/core/compat/arg_map_context.h index 122ebed21942a..0c6fdcb13912f 100644 --- a/paddle/phi/core/compat/arg_map_context.h +++ b/paddle/phi/core/compat/arg_map_context.h @@ -27,32 +27,62 @@ limitations under the License. */ namespace phi { // tuple(input_names, attr_names, output_names) -using KernelArgsTuple = std::tuple, - paddle::SmallVector, - paddle::SmallVector>; +using KernelArgsTuple = std::tuple, + paddle::small_vector, + paddle::small_vector>; struct KernelSignature { const char* name; - KernelArgsTuple args; + paddle::small_vector input_names; + paddle::small_vector attr_names; + paddle::small_vector output_names; KernelSignature() = default; KernelSignature(const char* kernel_name, - paddle::SmallVector&& inputs, - paddle::SmallVector&& attrs, - paddle::SmallVector&& outputs) - : name(kernel_name), args(std::make_tuple(inputs, attrs, outputs)) {} + paddle::small_vector&& inputs, + paddle::small_vector&& attrs, + paddle::small_vector&& outputs) + : name(kernel_name), + input_names(std::move(inputs)), + attr_names(std::move(attrs)), + output_names(std::move(outputs)) {} KernelSignature(const char* kernel_name, - const paddle::SmallVector& inputs, - const paddle::SmallVector& attrs, - const paddle::SmallVector& outputs) - : name(kernel_name), args(std::make_tuple(inputs, attrs, outputs)) {} + const paddle::small_vector& inputs, + const paddle::small_vector& attrs, + const paddle::small_vector& outputs) + : name(kernel_name), + input_names(inputs), + attr_names(attrs), + output_names(outputs) {} // TODO(chenweihang): add assign constructor to solve windows compile // problem, remove it later + KernelSignature(const KernelSignature& other) + : name(other.name), + input_names(other.input_names), + attr_names(other.attr_names), + output_names(other.output_names) {} + + KernelSignature(KernelSignature&& other) noexcept + : name(other.name), + input_names(std::move(other.input_names)), + attr_names(std::move(other.attr_names)), + output_names(std::move(other.output_names)) {} + KernelSignature& operator=(const KernelSignature& other) { name = other.name; - args = other.args; + input_names = other.input_names; + attr_names = other.attr_names; + output_names = other.output_names; + return *this; + } + + KernelSignature& operator=(KernelSignature&& other) noexcept { + name = other.name; + input_names.swap(other.input_names); + attr_names.swap(other.attr_names); + output_names.swap(other.output_names); return *this; } }; @@ -76,6 +106,7 @@ class ArgumentMappingContext { virtual size_t OutputSize(const std::string& name) const = 0; virtual bool IsDenseTensorInput(const std::string& name) const = 0; + virtual bool IsDenseTensorInputs(const std::string& name) const = 0; virtual bool IsSelectedRowsInput(const std::string& name) const = 0; // For compatibility with LoDTensorArray virtual bool IsDenseTensorVectorInput(const std::string& name) const = 0; diff --git a/paddle/phi/core/compat/convert_utils.cc b/paddle/phi/core/compat/convert_utils.cc index 4388bd1f751cf..18c39bfae1d18 100644 --- a/paddle/phi/core/compat/convert_utils.cc +++ b/paddle/phi/core/compat/convert_utils.cc @@ -28,27 +28,28 @@ namespace phi { Backend TransToPhiBackend(const phi::Place& place) { auto allocation_type = place.GetType(); - if (allocation_type == phi::AllocationType::CPU) { - return Backend::CPU; - } else if (allocation_type == phi::AllocationType::GPU) { - return Backend::GPU; - } else if (allocation_type == phi::AllocationType::GPUPINNED) { - return Backend::GPU; - } else if (allocation_type == phi::AllocationType::XPU) { - return Backend::XPU; - } else if (allocation_type == phi::AllocationType::NPU) { - return Backend::NPU; - } else if (allocation_type == phi::AllocationType::IPU) { - return Backend::IPU; - } else if (allocation_type == phi::AllocationType::MLU) { - return Backend::MLU; - } else if (allocation_type == phi::AllocationType::CUSTOM) { - return static_cast( - static_cast(Backend::NUM_BACKENDS) + - GetOrRegisterGlobalDeviceTypeId(place.GetDeviceType())); - } else { - PADDLE_THROW(phi::errors::InvalidArgument( - "Unsupported transform %s to phi Backend.", place)); + switch (allocation_type) { + case phi::AllocationType::GPU: + return Backend::GPU; + case AllocationType::CPU: + return Backend::CPU; + case AllocationType::GPUPINNED: + return Backend::GPU; + case AllocationType::XPU: + return Backend::XPU; + case AllocationType::NPU: + return Backend::NPU; + case AllocationType::IPU: + return Backend::IPU; + case AllocationType::MLU: + return Backend::MLU; + case AllocationType::CUSTOM: + return static_cast( + static_cast(Backend::NUM_BACKENDS) + + GetOrRegisterGlobalDeviceTypeId(place.GetDeviceType())); + default: + PADDLE_THROW(phi::errors::InvalidArgument( + "Unsupported transform %s to phi Backend.", place)); } } diff --git a/paddle/phi/core/compat/op_utils.h b/paddle/phi/core/compat/op_utils.h index 9c926fa871b67..bd19d403c9406 100644 --- a/paddle/phi/core/compat/op_utils.h +++ b/paddle/phi/core/compat/op_utils.h @@ -86,6 +86,14 @@ class DefaultKernelSignatureMap { return it->second; } + const KernelSignature* GetNullable(const std::string& op_type) const { + auto it = map_.find(op_type); + if (it != map_.end()) { + return &it->second; + } + return nullptr; + } + void Insert(std::string op_type, KernelSignature signature) { PADDLE_ENFORCE_NE( Has(op_type), @@ -148,16 +156,13 @@ class OpUtilsMap { } } - ArgumentMappingFn GetArgumentMappingFn(const std::string& op_type) const { + const ArgumentMappingFn* GetArgumentMappingFn( + const std::string& op_type) const { auto it = arg_mapping_fn_map_.find(op_type); if (it == arg_mapping_fn_map_.end()) { - auto func = - [&op_type](const ArgumentMappingContext& ctx) -> KernelSignature { - return DefaultKernelSignatureMap::Instance().Get(op_type); - }; - return func; + return nullptr; } else { - return it->second; + return &it->second; } } diff --git a/paddle/phi/core/dense_tensor.cc b/paddle/phi/core/dense_tensor.cc index 2b9a5f5e0ea0c..6c9291f816f7a 100644 --- a/paddle/phi/core/dense_tensor.cc +++ b/paddle/phi/core/dense_tensor.cc @@ -135,7 +135,6 @@ void* DenseTensor::AllocateFrom(Allocator* allocator, template const T* DenseTensor::data() const { - check_memory_size(); PADDLE_ENFORCE_EQ( dtype(), paddle::experimental::CppTypeToDataType::Type(), @@ -147,13 +146,13 @@ const T* DenseTensor::data() const { template T* DenseTensor::data() { - check_memory_size(); + T* ret = static_cast(data()); PADDLE_ENFORCE( (dtype() == paddle::experimental::CppTypeToDataType::Type()), phi::errors::InvalidArgument( "The type of data we are trying to retrieve does not match the " "type of data currently contained in the container.")); - return static_cast(data()); + return ret; } void* DenseTensor::data() { diff --git a/paddle/phi/core/infermeta_utils.cc b/paddle/phi/core/infermeta_utils.cc index 70f26102cbad1..1d61f55f9dcd2 100644 --- a/paddle/phi/core/infermeta_utils.cc +++ b/paddle/phi/core/infermeta_utils.cc @@ -30,12 +30,12 @@ void InferMetaContext::EmplaceBackOutput(MetaTensor output) { outputs_.emplace_back(std::move(output)); output_range_.emplace_back(std::pair(index, index + 1)); } -void InferMetaContext::EmplaceBackAttr(paddle::any attr) { +void InferMetaContext::EmplaceBackAttr(Attribute attr) { attrs_.emplace_back(std::move(attr)); } void InferMetaContext::EmplaceBackInputs( - paddle::SmallVector inputs) { + paddle::small_vector inputs) { int index = inputs_.size(); input_range_.emplace_back(std::pair(index, index + inputs.size())); inputs_.insert(inputs_.end(), @@ -43,7 +43,7 @@ void InferMetaContext::EmplaceBackInputs( std::make_move_iterator(inputs.end())); } void InferMetaContext::EmplaceBackOutputs( - paddle::SmallVector outputs) { + paddle::small_vector outputs) { int index = outputs_.size(); output_range_.emplace_back( std::pair(index, index + outputs.size())); @@ -120,6 +120,38 @@ std::vector InferMetaContext::MutableOutputBetween(size_t start, return result; } +template +const AttrType& InferMetaContext::AttrAt(size_t idx) const { + try { + return paddle::get(attrs_.at(idx)); + } catch (paddle::bad_variant_access const& e) { + PADDLE_THROW(phi::errors::InvalidArgument( + "Attribute cast error in InferMeta Context, the expected attribute " + "type is `%s`.", + std::type_index(typeid(AttrType)).name())); + } +} + +template const bool& InferMetaContext::AttrAt(size_t idx) const; +template const int& InferMetaContext::AttrAt(size_t idx) const; +template const int64_t& InferMetaContext::AttrAt(size_t idx) const; +template const float& InferMetaContext::AttrAt(size_t idx) const; +template const double& InferMetaContext::AttrAt(size_t idx) const; +template const std::string& InferMetaContext::AttrAt(size_t idx) const; +template const std::vector& InferMetaContext::AttrAt(size_t idx) const; +template const std::vector& InferMetaContext::AttrAt(size_t idx) const; +template const std::vector& InferMetaContext::AttrAt(size_t idx) const; +template const std::vector& InferMetaContext::AttrAt(size_t idx) const; +template const std::vector& InferMetaContext::AttrAt(size_t idx) const; +template const std::vector& InferMetaContext::AttrAt( + size_t idx) const; +template const Scalar& InferMetaContext::AttrAt(size_t idx) const; +template const std::vector& InferMetaContext::AttrAt(size_t idx) const; +template const IntArray& InferMetaContext::AttrAt(size_t idx) const; +template const DataType& InferMetaContext::AttrAt(size_t idx) const; +template const DataLayout& InferMetaContext::AttrAt(size_t idx) const; +template const Place& InferMetaContext::AttrAt(size_t idx) const; + MetaFnFactory& MetaFnFactory::Instance() { static MetaFnFactory g_meta_fn_map; return g_meta_fn_map; diff --git a/paddle/phi/core/infermeta_utils.h b/paddle/phi/core/infermeta_utils.h index 699c38ebd4702..b974f2c868a8a 100644 --- a/paddle/phi/core/infermeta_utils.h +++ b/paddle/phi/core/infermeta_utils.h @@ -21,6 +21,7 @@ limitations under the License. */ #include "paddle/phi/common/int_array.h" #include "paddle/phi/common/scalar.h" +#include "paddle/phi/core/attribute.h" #include "paddle/phi/core/enforce.h" #include "paddle/phi/core/macros.h" #include "paddle/phi/core/meta_tensor.h" @@ -41,12 +42,12 @@ class InferMetaContext { void EmplaceBackInput(MetaTensor input); void EmplaceBackOutput(MetaTensor output); - void EmplaceBackAttr(paddle::any attr); + void EmplaceBackAttr(Attribute attr); void EmplaceBackInputs( - paddle::SmallVector inputs); + paddle::small_vector inputs); void EmplaceBackOutputs( - paddle::SmallVector outputs); + paddle::small_vector outputs); virtual const MetaTensor& InputAt(size_t idx) const; virtual paddle::optional OptionalInputAt(size_t idx) const; @@ -61,17 +62,7 @@ class InferMetaContext { size_t end); template - AttrType AttrAt(size_t idx) { - try { - return paddle::any_cast(attrs_.at(idx)); - } catch (paddle::bad_any_cast& e) { - PADDLE_THROW(phi::errors::InvalidArgument( - "Attribute cast error in InferMeta Context, the expected attribute " - "type is `%s`, but actual attribute type is `%s`.", - std::type_index(typeid(AttrType)).name(), - std::type_index(attrs_.at(idx).type()).name())); - } - } + const AttrType& AttrAt(size_t idx) const; const std::pair& InputRangeAt(size_t idx) const; const std::pair& OutputRangeAt(size_t idx) const; @@ -81,16 +72,16 @@ class InferMetaContext { protected: MetaConfig config_; - paddle::SmallVector attrs_; + paddle::small_vector attrs_; - paddle::SmallVector, phi::kInputSmallVectorSize> + paddle::small_vector, phi::kInputSmallVectorSize> input_range_; - paddle::SmallVector, phi::kOutputSmallVectorSize> + paddle::small_vector, phi::kOutputSmallVectorSize> output_range_; private: - paddle::SmallVector inputs_; - paddle::SmallVector outputs_; + paddle::small_vector inputs_; + paddle::small_vector outputs_; }; #define PD_INFER_META(...) \ @@ -111,6 +102,21 @@ class InferMetaContext { } \ } +#define PD_SPECIALIZE_InferMetaFnCallHelper_FOR_CONST_ATTRIBUTE_REF(attr_type) \ + template \ + struct InferMetaFnCallHelper { \ + template \ + static void Call(InferMetaContext* ctx, PreviousArgs&... pargs) { \ + static_assert(out_idx == 0, \ + "InferMeta's Attributes should appear before Outputs."); \ + const attr_type& arg = ctx->AttrAt(attr_idx); \ + InferMetaFnCallHelper< \ + Tail...>::template Call(ctx, \ + pargs..., \ + arg); \ + } \ + } + template struct InferMetaTypeTag {}; @@ -201,27 +207,27 @@ struct InferMetaFnImpl { } }; - // TODO(chenweihang): support other attr type later PD_SPECIALIZE_InferMetaFnCallHelper_FOR_ATTRIBUTE(bool); PD_SPECIALIZE_InferMetaFnCallHelper_FOR_ATTRIBUTE(int); PD_SPECIALIZE_InferMetaFnCallHelper_FOR_ATTRIBUTE(int64_t); PD_SPECIALIZE_InferMetaFnCallHelper_FOR_ATTRIBUTE(float); - PD_SPECIALIZE_InferMetaFnCallHelper_FOR_ATTRIBUTE(const std::string&); - PD_SPECIALIZE_InferMetaFnCallHelper_FOR_ATTRIBUTE(const std::vector&); - PD_SPECIALIZE_InferMetaFnCallHelper_FOR_ATTRIBUTE(const std::vector&); - PD_SPECIALIZE_InferMetaFnCallHelper_FOR_ATTRIBUTE( - const std::vector&); - PD_SPECIALIZE_InferMetaFnCallHelper_FOR_ATTRIBUTE(const std::vector&); - PD_SPECIALIZE_InferMetaFnCallHelper_FOR_ATTRIBUTE(const std::vector&); - PD_SPECIALIZE_InferMetaFnCallHelper_FOR_ATTRIBUTE( - const std::vector&); PD_SPECIALIZE_InferMetaFnCallHelper_FOR_ATTRIBUTE(DataType); PD_SPECIALIZE_InferMetaFnCallHelper_FOR_ATTRIBUTE(Backend); PD_SPECIALIZE_InferMetaFnCallHelper_FOR_ATTRIBUTE(DataLayout); - PD_SPECIALIZE_InferMetaFnCallHelper_FOR_ATTRIBUTE(const Scalar&); - PD_SPECIALIZE_InferMetaFnCallHelper_FOR_ATTRIBUTE(const IntArray&); - - // TODO(chenweihang): support vector input later + PD_SPECIALIZE_InferMetaFnCallHelper_FOR_CONST_ATTRIBUTE_REF(std::string); + PD_SPECIALIZE_InferMetaFnCallHelper_FOR_CONST_ATTRIBUTE_REF(Scalar); + PD_SPECIALIZE_InferMetaFnCallHelper_FOR_CONST_ATTRIBUTE_REF(IntArray); + PD_SPECIALIZE_InferMetaFnCallHelper_FOR_CONST_ATTRIBUTE_REF( + std::vector); + PD_SPECIALIZE_InferMetaFnCallHelper_FOR_CONST_ATTRIBUTE_REF(std::vector); + PD_SPECIALIZE_InferMetaFnCallHelper_FOR_CONST_ATTRIBUTE_REF( + std::vector); + PD_SPECIALIZE_InferMetaFnCallHelper_FOR_CONST_ATTRIBUTE_REF( + std::vector); + PD_SPECIALIZE_InferMetaFnCallHelper_FOR_CONST_ATTRIBUTE_REF( + std::vector); + PD_SPECIALIZE_InferMetaFnCallHelper_FOR_CONST_ATTRIBUTE_REF( + std::vector); template struct InferMetaFnCallHelper { diff --git a/paddle/phi/core/kernel_context.cc b/paddle/phi/core/kernel_context.cc index cf862cbde18f9..c902fc824f8d2 100644 --- a/paddle/phi/core/kernel_context.cc +++ b/paddle/phi/core/kernel_context.cc @@ -28,7 +28,7 @@ void KernelContext::EmplaceBackInputWithoutSetRange(const TensorBase* input) { } void KernelContext::EmplaceBackInputs( - paddle::SmallVector inputs) { + paddle::small_vector inputs) { int index = inputs_.size(); // Record the start and end index of the input input_range_.emplace_back(std::pair(index, index + inputs.size())); @@ -38,7 +38,7 @@ void KernelContext::EmplaceBackInputs( } void KernelContext::EmplaceBackInputsWithoutSetRange( - paddle::SmallVector inputs) { + paddle::small_vector inputs) { inputs_.insert(inputs_.end(), std::make_move_iterator(inputs.begin()), std::make_move_iterator(inputs.end())); @@ -56,7 +56,7 @@ void KernelContext::EmplaceBackOutputWithoutSetRange(TensorBase* output) { } void KernelContext::EmplaceBackOutputs( - paddle::SmallVector outputs) { + paddle::small_vector outputs) { int index = outputs_.size(); // Record the start and end index of the input output_range_.emplace_back( @@ -67,13 +67,13 @@ void KernelContext::EmplaceBackOutputs( } void KernelContext::EmplaceBackOutputsWithoutSetRange( - paddle::SmallVector outputs) { + paddle::small_vector outputs) { outputs_.insert(outputs_.end(), std::make_move_iterator(outputs.begin()), std::make_move_iterator(outputs.end())); } -void KernelContext::EmplaceBackAttr(paddle::any attr) { +void KernelContext::EmplaceBackAttr(Attribute attr) { attrs_.emplace_back(std::move(attr)); } @@ -113,4 +113,34 @@ const std::pair& KernelContext::OutputRangeAt(size_t idx) const { return output_range_.at(idx); } +template +const AttrType& KernelContext::AttrAt(size_t idx) const { + try { + return paddle::get(attrs_.at(idx)); + } catch (paddle::bad_variant_access const& ex) { + PADDLE_THROW(phi::errors::InvalidArgument( + "Attribute cast error in Op Kernel Context.")); + } +} + +template const bool& KernelContext::AttrAt(size_t idx) const; +template const int& KernelContext::AttrAt(size_t idx) const; +template const int64_t& KernelContext::AttrAt(size_t idx) const; +template const float& KernelContext::AttrAt(size_t idx) const; +template const double& KernelContext::AttrAt(size_t idx) const; +template const std::string& KernelContext::AttrAt(size_t idx) const; +template const std::vector& KernelContext::AttrAt(size_t idx) const; +template const std::vector& KernelContext::AttrAt(size_t idx) const; +template const std::vector& KernelContext::AttrAt(size_t idx) const; +template const std::vector& KernelContext::AttrAt(size_t idx) const; +template const std::vector& KernelContext::AttrAt(size_t idx) const; +template const std::vector& KernelContext::AttrAt( + size_t idx) const; +template const Scalar& KernelContext::AttrAt(size_t idx) const; +template const std::vector& KernelContext::AttrAt(size_t idx) const; +template const IntArray& KernelContext::AttrAt(size_t idx) const; +template const DataType& KernelContext::AttrAt(size_t idx) const; +template const DataLayout& KernelContext::AttrAt(size_t idx) const; +template const Place& KernelContext::AttrAt(size_t idx) const; + } // namespace phi diff --git a/paddle/phi/core/kernel_context.h b/paddle/phi/core/kernel_context.h index ab4e044e62537..8b43239d352b3 100644 --- a/paddle/phi/core/kernel_context.h +++ b/paddle/phi/core/kernel_context.h @@ -17,11 +17,12 @@ #include #include +#include "paddle/phi/core/attribute.h" #include "paddle/phi/core/device_context.h" #include "paddle/phi/core/enforce.h" #include "paddle/phi/core/tensor_base.h" #include "paddle/phi/core/tensor_utils.h" -#include "paddle/utils/any.h" +#include "paddle/phi/core/type_defs.h" #include "paddle/utils/optional.h" #include "paddle/utils/small_vector.h" @@ -50,21 +51,21 @@ class KernelContext { void EmplaceBackInputWithoutSetRange(const TensorBase* input); - void EmplaceBackInputs(paddle::SmallVector inputs); + void EmplaceBackInputs(paddle::small_vector inputs); void EmplaceBackInputsWithoutSetRange( - paddle::SmallVector inputs); + paddle::small_vector inputs); void EmplaceBackOutput(TensorBase* output); void EmplaceBackOutputWithoutSetRange(TensorBase* output); - void EmplaceBackOutputs(paddle::SmallVector outputs); + void EmplaceBackOutputs(paddle::small_vector outputs); void EmplaceBackOutputsWithoutSetRange( - paddle::SmallVector outputs); + paddle::small_vector outputs); - void EmplaceBackAttr(paddle::any attr); + void EmplaceBackAttr(Attribute attr); const std::pair& InputRangeAt(size_t idx) const; @@ -128,14 +129,7 @@ class KernelContext { } template - AttrType AttrAt(size_t idx) const { - try { - return paddle::any_cast(attrs_.at(idx)); - } catch (paddle::bad_any_cast&) { - PADDLE_THROW(phi::errors::InvalidArgument( - "Attribute cast error in Op Kernel Context.")); - } - } + const AttrType& AttrAt(size_t idx) const; size_t InputsSize() const { return inputs_.size(); } size_t OutputsSize() const { return outputs_.size(); } @@ -144,12 +138,13 @@ class KernelContext { private: DeviceContext* dev_ctx_; - paddle::SmallVector inputs_; - paddle::SmallVector outputs_; - paddle::SmallVector attrs_; + paddle::small_vector inputs_; + paddle::small_vector outputs_; + paddle::small_vector attrs_; - paddle::SmallVector> input_range_; - paddle::SmallVector> output_range_; + paddle::small_vector, kInputSmallVectorSize> input_range_; + paddle::small_vector, kOutputSmallVectorSize> + output_range_; }; } // namespace phi diff --git a/paddle/phi/core/kernel_factory.cc b/paddle/phi/core/kernel_factory.cc index d3fd2e0204e54..08329d0c8636a 100644 --- a/paddle/phi/core/kernel_factory.cc +++ b/paddle/phi/core/kernel_factory.cc @@ -79,7 +79,7 @@ bool KernelFactory::HasKernel(const std::string& kernel_name, const Kernel& KernelFactory::SelectKernelOrThrowError( const std::string& kernel_name, const KernelKey& kernel_key, - bool use_cudnn) const { + bool use_gpudnn) const { auto iter = kernels_.find(kernel_name); PADDLE_ENFORCE_NE( iter, @@ -87,7 +87,7 @@ const Kernel& KernelFactory::SelectKernelOrThrowError( phi::errors::NotFound("The kernel `%s` is not registered.", kernel_name)); #if defined(PADDLE_WITH_CUDA) || defined(PADDLE_WITH_HIP) - if (use_cudnn && kernel_key.backend() == Backend::GPU) { + if (use_gpudnn && kernel_key.backend() == Backend::GPU) { auto kernel_iter = iter->second.find( {Backend::GPUDNN, kernel_key.layout(), kernel_key.dtype()}); if (kernel_iter == iter->second.end() && @@ -140,6 +140,68 @@ const KernelArgsDef& KernelFactory::GetFirstKernelArgsDef( return iter->second.cbegin()->second.args_def(); } +std::ostream& operator<<(std::ostream& os, AttributeType attr_type) { + switch (attr_type) { + case AttributeType::BOOL: + os << "bool"; + break; + case AttributeType::INT32: + os << "int"; + break; + case AttributeType::INT64: + os << "int64_t"; + break; + case AttributeType::FLOAT32: + os << "float"; + break; + case AttributeType::FLOAT64: + os << "double"; + break; + case AttributeType::STRING: + os << "string"; + break; + case AttributeType::BOOLS: + os << "vector"; + break; + case AttributeType::INT32S: + os << "vector"; + break; + case AttributeType::INT64S: + os << "vector"; + break; + case AttributeType::FLOAT32S: + os << "vector"; + break; + case AttributeType::FLOAT64S: + os << "vector"; + break; + case AttributeType::STRINGS: + os << "vector"; + break; + case AttributeType::SCALAR: + os << "Scalar"; + break; + case AttributeType::SCALARS: + os << "vector"; + break; + case AttributeType::INT_ARRAY: + os << "IntArray"; + break; + case AttributeType::DATA_TYPE: + os << "DataType"; + break; + case AttributeType::DATA_LAYOUT: + os << "DataLayout"; + break; + case AttributeType::PLACE: + os << "Place"; + break; + default: + os << "Undefined"; + } + return os; +} + // print kernel info with json format: // { // "(CPU, Undefined(AnyLayout), complex64)": { @@ -175,7 +237,7 @@ std::ostream& operator<<(std::ostream& os, const Kernel& kernel) { need_comma = false; for (auto& arg_def : kernel.args_def().attribute_defs()) { if (need_comma) os << ","; - os << "\"" << arg_def.type_index.name() << "\""; + os << "\"" << arg_def.type_index << "\""; need_comma = true; } os << "]}"; diff --git a/paddle/phi/core/kernel_factory.h b/paddle/phi/core/kernel_factory.h index 812b6222cb5e2..c4c8274db976c 100644 --- a/paddle/phi/core/kernel_factory.h +++ b/paddle/phi/core/kernel_factory.h @@ -122,11 +122,33 @@ struct TensorArgDef { } }; +// Align the original fluid Attribute type with lower overhead +enum class AttributeType { + UNDEFINED = 0, + BOOL, + INT32, + INT64, + FLOAT32, + FLOAT64, + STRING, + BOOLS, + INT32S, + INT64S, + FLOAT32S, + FLOAT64S, + STRINGS, + SCALAR, + SCALARS, + INT_ARRAY, + DATA_TYPE, + DATA_LAYOUT, + PLACE, +}; + struct AttributeArgDef { - std::type_index type_index; + AttributeType type_index; - explicit AttributeArgDef(std::type_index type_index) - : type_index(type_index) {} + explicit AttributeArgDef(AttributeType type_index) : type_index(type_index) {} }; class KernelArgsDef { @@ -147,41 +169,42 @@ class KernelArgsDef { output_defs_.emplace_back(TensorArgDef(backend, layout, dtype, type_index)); } - void AppendAttribute(std::type_index type_index) { + void AppendAttribute(AttributeType type_index) { attribute_defs_.emplace_back(AttributeArgDef(type_index)); } - const paddle::SmallVector& input_defs() + const paddle::small_vector& input_defs() const { return input_defs_; } - const paddle::SmallVector& output_defs() - const { + const paddle::small_vector& + output_defs() const { return output_defs_; } - const paddle::SmallVector& + const paddle::small_vector& attribute_defs() const { return attribute_defs_; } - paddle::SmallVector& input_defs() { + paddle::small_vector& input_defs() { return input_defs_; } - paddle::SmallVector& output_defs() { + paddle::small_vector& output_defs() { return output_defs_; } - paddle::SmallVector& attribute_defs() { + paddle::small_vector& + attribute_defs() { return attribute_defs_; } private: - paddle::SmallVector input_defs_{{}}; - paddle::SmallVector output_defs_{{}}; - paddle::SmallVector attribute_defs_{ + paddle::small_vector input_defs_{{}}; + paddle::small_vector output_defs_{{}}; + paddle::small_vector attribute_defs_{ {}}; }; @@ -247,7 +270,7 @@ class KernelFactory { const Kernel& SelectKernelOrThrowError(const std::string& kernel_name, const KernelKey& kernel_key, - bool use_cudnn = false) const; + bool use_gpudnn = false) const; const Kernel& SelectKernelOrThrowError(const std::string& kernel_name, Backend backend, @@ -277,6 +300,8 @@ inline std::ostream& operator<<(std::ostream& os, const KernelKey& kernel_key) { return os; } +std::ostream& operator<<(std::ostream& os, AttributeType attr_type); + std::ostream& operator<<(std::ostream& os, const Kernel& kernel); std::ostream& operator<<(std::ostream& os, KernelFactory& kernel_factory); diff --git a/paddle/phi/core/kernel_registry.h b/paddle/phi/core/kernel_registry.h index b18fd9e05f92f..36ab9c081cc37 100644 --- a/paddle/phi/core/kernel_registry.h +++ b/paddle/phi/core/kernel_registry.h @@ -105,6 +105,11 @@ struct KernelArgsParseFunctor { default_tensor_layout, default_key.dtype(), arg_type); + } else if (arg_type == std::type_index(typeid(const StringTensor&))) { + args_def->AppendInput(default_key.backend(), + default_tensor_layout, + default_key.dtype(), + arg_type); } else if (arg_type == std::type_index(typeid(const SparseCooTensor&))) { args_def->AppendInput(default_key.backend(), default_tensor_layout, @@ -153,11 +158,56 @@ struct KernelArgsParseFunctor { default_tensor_layout, default_key.dtype(), arg_type); + } else if (arg_type == std::type_index(typeid(StringTensor*))) { + args_def->AppendOutput(default_key.backend(), + default_tensor_layout, + default_key.dtype(), + arg_type); + } else if (arg_type == std::type_index(typeid(bool))) { + args_def->AppendAttribute(AttributeType::BOOL); + } else if (arg_type == std::type_index(typeid(int))) { + args_def->AppendAttribute(AttributeType::INT32); + } else if (arg_type == std::type_index(typeid(int64_t))) { + args_def->AppendAttribute(AttributeType::INT64); + } else if (arg_type == std::type_index(typeid(float))) { + args_def->AppendAttribute(AttributeType::FLOAT32); + } else if (arg_type == std::type_index(typeid(double))) { + args_def->AppendAttribute(AttributeType::FLOAT64); + } else if (arg_type == std::type_index(typeid(std::string))) { + args_def->AppendAttribute(AttributeType::STRING); + } else if (arg_type == + std::type_index(typeid(const std::vector&))) { + args_def->AppendAttribute(AttributeType::BOOLS); + } else if (arg_type == std::type_index(typeid(const std::vector&))) { + args_def->AppendAttribute(AttributeType::INT32S); + } else if (arg_type == + std::type_index(typeid(const std::vector&))) { + args_def->AppendAttribute(AttributeType::INT64S); + } else if (arg_type == + std::type_index(typeid(const std::vector&))) { + args_def->AppendAttribute(AttributeType::FLOAT32S); + } else if (arg_type == + std::type_index(typeid(const std::vector&))) { + args_def->AppendAttribute(AttributeType::FLOAT64S); + } else if (arg_type == + std::type_index(typeid(const std::vector&))) { + args_def->AppendAttribute(AttributeType::STRINGS); + } else if (arg_type == std::type_index(typeid(const Scalar&))) { + args_def->AppendAttribute(AttributeType::SCALAR); + } else if (arg_type == + std::type_index(typeid(const std::vector&))) { + args_def->AppendAttribute(AttributeType::SCALARS); + } else if (arg_type == std::type_index(typeid(const IntArray&))) { + args_def->AppendAttribute(AttributeType::INT_ARRAY); + } else if (arg_type == std::type_index(typeid(DataType))) { + args_def->AppendAttribute(AttributeType::DATA_TYPE); + } else if (arg_type == std::type_index(typeid(DataLayout))) { + args_def->AppendAttribute(AttributeType::DATA_LAYOUT); + } else if (arg_type == std::type_index(typeid(Place))) { + args_def->AppendAttribute(AttributeType::PLACE); } else { - // Attribute deal with - // TODO(chenweihang): now here allow any types of attribute, maybe - // should add limits here - args_def->AppendAttribute(arg_type); + PADDLE_THROW(phi::errors::Unavailable( + "Unsupported kernel argument type `%s`.", arg_type.name())); } } } diff --git a/paddle/phi/core/kernel_utils.h b/paddle/phi/core/kernel_utils.h index 55574ea03ab4a..f548d1da2d4e7 100644 --- a/paddle/phi/core/kernel_utils.h +++ b/paddle/phi/core/kernel_utils.h @@ -75,7 +75,7 @@ namespace phi { "Kernel's Input should appear before Attributes."); \ static_assert(out_idx == 0, \ "Kernel's Input should appear before Outputs."); \ - const std::pair range = ctx->InputRangeAt(in_idx); \ + const std::pair& range = ctx->InputRangeAt(in_idx); \ const tensor_type& arg = ctx->InputAt(range.first); \ KernelCallHelper:: \ template Compute( \ @@ -96,7 +96,7 @@ namespace phi { "Kernel's Input should appear before Attributes."); \ static_assert(out_idx == 0, \ "Kernel's Input should appear before Outputs."); \ - const std::pair range = ctx->InputRangeAt(in_idx); \ + const std::pair& range = ctx->InputRangeAt(in_idx); \ auto arg = ctx->OptionalInputAt(range.first); \ KernelCallHelper:: \ template Compute( \ @@ -117,7 +117,7 @@ namespace phi { "Kernel's Input should appear before Attributes."); \ static_assert(out_idx == 0, \ "Kernel's Input should appear before Outputs."); \ - const std::pair range = ctx->InputRangeAt(in_idx); \ + const std::pair& range = ctx->InputRangeAt(in_idx); \ std::vector arg = std::move( \ ctx->InputsBetween(range.first, range.second)); \ KernelCallHelper:: \ @@ -141,7 +141,7 @@ namespace phi { "Kernel's Input should appear before Attributes."); \ static_assert(out_idx == 0, \ "Kernel's Input should appear before Outputs."); \ - const std::pair range = ctx->InputRangeAt(in_idx); \ + const std::pair& range = ctx->InputRangeAt(in_idx); \ paddle::optional> arg = \ ctx->OptionalInputsBetween(range.first, range.second); \ KernelCallHelper:: \ @@ -168,6 +168,24 @@ namespace phi { } \ } +#define PD_SPECIALIZE_KernelCallHelper_FOR_CONST_ATTRIBUTE_REF(attr_type) \ + template \ + struct KernelCallHelper { \ + template \ + static void Compute(KernelContext* ctx, PreviousArgs&... pargs) { \ + static_assert(out_idx == 0, \ + "Kernel's Attributes should appear before Outputs."); \ + const attr_type& arg = ctx->AttrAt(attr_idx); \ + KernelCallHelper:: \ + template Compute( \ + ctx, pargs..., arg); \ + } \ + } + #define PD_SPECIALIZE_KernelCallHelper_FOR_OUTPUT(tensor_type) \ template \ struct KernelCallHelper { \ @@ -177,7 +195,7 @@ namespace phi { int out_idx, \ typename... PreviousArgs> \ static void Compute(KernelContext* ctx, PreviousArgs&... pargs) { \ - const std::pair range = ctx->OutputRangeAt(out_idx); \ + const std::pair& range = ctx->OutputRangeAt(out_idx); \ tensor_type* arg = ctx->MutableOutputAt(range.first); \ KernelCallHelper:: \ template Compute( \ @@ -194,7 +212,7 @@ namespace phi { int out_idx, \ typename... PreviousArgs> \ static void Compute(KernelContext* ctx, PreviousArgs&... pargs) { \ - const std::pair range = ctx->OutputRangeAt(out_idx); \ + const std::pair& range = ctx->OutputRangeAt(out_idx); \ std::vector arg = std::move( \ ctx->MutableOutputBetween(range.first, range.second)); \ KernelCallHelper:: \ @@ -270,19 +288,20 @@ struct KernelImpl { PD_SPECIALIZE_KernelCallHelper_FOR_ATTRIBUTE(int); PD_SPECIALIZE_KernelCallHelper_FOR_ATTRIBUTE(int64_t); PD_SPECIALIZE_KernelCallHelper_FOR_ATTRIBUTE(phi::dtype::float16); - PD_SPECIALIZE_KernelCallHelper_FOR_ATTRIBUTE(const Scalar&); PD_SPECIALIZE_KernelCallHelper_FOR_ATTRIBUTE(DataType); PD_SPECIALIZE_KernelCallHelper_FOR_ATTRIBUTE(DataLayout); PD_SPECIALIZE_KernelCallHelper_FOR_ATTRIBUTE(Place); - PD_SPECIALIZE_KernelCallHelper_FOR_ATTRIBUTE(const std::vector&); - PD_SPECIALIZE_KernelCallHelper_FOR_ATTRIBUTE(const IntArray&); - PD_SPECIALIZE_KernelCallHelper_FOR_ATTRIBUTE(const std::vector&); - PD_SPECIALIZE_KernelCallHelper_FOR_ATTRIBUTE(const std::string&); - PD_SPECIALIZE_KernelCallHelper_FOR_ATTRIBUTE(const std::vector&); - PD_SPECIALIZE_KernelCallHelper_FOR_ATTRIBUTE(const std::vector&); - PD_SPECIALIZE_KernelCallHelper_FOR_ATTRIBUTE(const std::vector&); - PD_SPECIALIZE_KernelCallHelper_FOR_ATTRIBUTE(const std::vector&); - PD_SPECIALIZE_KernelCallHelper_FOR_ATTRIBUTE(const std::vector&); + PD_SPECIALIZE_KernelCallHelper_FOR_CONST_ATTRIBUTE_REF(std::string); + PD_SPECIALIZE_KernelCallHelper_FOR_CONST_ATTRIBUTE_REF(Scalar); + PD_SPECIALIZE_KernelCallHelper_FOR_CONST_ATTRIBUTE_REF(IntArray); + PD_SPECIALIZE_KernelCallHelper_FOR_CONST_ATTRIBUTE_REF(std::vector); + PD_SPECIALIZE_KernelCallHelper_FOR_CONST_ATTRIBUTE_REF(std::vector); + PD_SPECIALIZE_KernelCallHelper_FOR_CONST_ATTRIBUTE_REF(std::vector); + PD_SPECIALIZE_KernelCallHelper_FOR_CONST_ATTRIBUTE_REF(std::vector); + PD_SPECIALIZE_KernelCallHelper_FOR_CONST_ATTRIBUTE_REF(std::vector); + PD_SPECIALIZE_KernelCallHelper_FOR_CONST_ATTRIBUTE_REF( + std::vector); + PD_SPECIALIZE_KernelCallHelper_FOR_CONST_ATTRIBUTE_REF(std::vector); /* Output Helpers */ diff --git a/paddle/phi/core/sparse_csr_tensor.cc b/paddle/phi/core/sparse_csr_tensor.cc index ab9717a564eb5..447fab0e33c5b 100644 --- a/paddle/phi/core/sparse_csr_tensor.cc +++ b/paddle/phi/core/sparse_csr_tensor.cc @@ -27,9 +27,11 @@ SparseCsrTensor::SparseCsrTensor() { inline void check_shape(const DDim& dims) { bool valid = dims.size() == 2 || dims.size() == 3; - PADDLE_ENFORCE(valid, - phi::errors::InvalidArgument( - "the SparseCsrTensor only support 2-D Tensor.")); + PADDLE_ENFORCE( + valid, + phi::errors::InvalidArgument("the SparseCsrTensor only support 2-D or " + "3-D Tensor, but get %d-D Tensor", + dims.size())); } #define Check(non_zero_crows, non_zero_cols, non_zero_elements, dims) \ { \ diff --git a/paddle/phi/core/type_defs.h b/paddle/phi/core/type_defs.h index a1e7836088389..2edca98bfd951 100644 --- a/paddle/phi/core/type_defs.h +++ b/paddle/phi/core/type_defs.h @@ -15,6 +15,8 @@ #pragma once #include +#include +#include namespace phi { @@ -36,8 +38,8 @@ using ArgumentMappingFn = using InferMetaFn = void (*)(InferMetaContext* ctx); // Global SmallVector size setting -constexpr size_t kInputSmallVectorSize = 10U; -constexpr size_t kAttrSmallVectorSize = 10U; -constexpr size_t kOutputSmallVectorSize = 5U; +constexpr size_t kInputSmallVectorSize = 15U; +constexpr size_t kAttrSmallVectorSize = 15U; +constexpr size_t kOutputSmallVectorSize = 15U; } // namespace phi diff --git a/paddle/phi/core/utils/type_registry.h b/paddle/phi/core/utils/type_registry.h index f27c3db2275c3..5b64dbd01643e 100644 --- a/paddle/phi/core/utils/type_registry.h +++ b/paddle/phi/core/utils/type_registry.h @@ -50,7 +50,8 @@ template TypeInfo TypeRegistry::RegisterType(const std::string& type) { std::lock_guard guard(mutex_); assert(name_to_id_.find(type) == name_to_id_.end()); - assert(names_.size() < std::numeric_limits::max()); + assert(names_.size() < static_cast( + std::numeric_limits::max())); int8_t id = static_cast(names_.size()); names_.emplace_back(type); name_to_id_[type] = id; diff --git a/paddle/phi/infermeta/backward.cc b/paddle/phi/infermeta/backward.cc index 567f39a915c02..602942abf4d34 100644 --- a/paddle/phi/infermeta/backward.cc +++ b/paddle/phi/infermeta/backward.cc @@ -67,6 +67,22 @@ void BilinearTensorProductGradInferMeta(const MetaTensor& x, } } +void ChannelShuffleGradInferMeta(const MetaTensor& out_grad, + int groups, + const std::string& data_format, + MetaTensor* x_grad) { + auto do_dims = out_grad.dims(); + PADDLE_ENFORCE_EQ(do_dims.size(), + 4, + phi::errors::InvalidArgument( + "Input should be a 4-D tensor of format [N, C, H, W] " + "or [N, H, W, C], but got %u.", + do_dims.size())); + auto dx_dims = do_dims; + x_grad->set_dims(dx_dims); + x_grad->set_dtype(out_grad.dtype()); +} + void ConvTransposeGradInferMeta(const MetaTensor& x, const MetaTensor& filter, const MetaTensor& dout, @@ -427,6 +443,36 @@ void NllLossGradInferMeta(const MetaTensor& x, } } +void PixelUnshuffleGradInferMeta(const MetaTensor& out_grad, + int downscale_factor, + const std::string& data_format, + MetaTensor* x_grad) { + auto do_dims = out_grad.dims(); + PADDLE_ENFORCE_EQ(do_dims.size(), + 4, + phi::errors::InvalidArgument( + "Input should be a 4-D tensor of format [N, C, H, W] " + "or [N, H, W, C], but got %u.", + do_dims.size())); + + const bool channel_last = (data_format == "NHWC"); + + auto dx_dims = do_dims; + dx_dims[0] = do_dims[0]; + + if (!channel_last) { + dx_dims[1] = do_dims[1] / (downscale_factor * downscale_factor); + dx_dims[2] = do_dims[2] * downscale_factor; + dx_dims[3] = do_dims[3] * downscale_factor; + } else { + dx_dims[1] = do_dims[1] * downscale_factor; + dx_dims[2] = do_dims[2] * downscale_factor; + dx_dims[3] = do_dims[3] / (downscale_factor * downscale_factor); + } + x_grad->set_dims(dx_dims); + x_grad->set_dtype(out_grad.dtype()); +} + void PoolGradInferMeta(const MetaTensor& x, const MetaTensor& out, const MetaTensor& dout, diff --git a/paddle/phi/infermeta/backward.h b/paddle/phi/infermeta/backward.h index 6807438ebbb75..c35b58d0f56e4 100644 --- a/paddle/phi/infermeta/backward.h +++ b/paddle/phi/infermeta/backward.h @@ -37,6 +37,11 @@ void BilinearTensorProductGradInferMeta(const MetaTensor& x, MetaTensor* dweight, MetaTensor* dbias); +void ChannelShuffleGradInferMeta(const MetaTensor& out_grad, + int groups, + const std::string& data_format, + MetaTensor* x_grad); + void ConvTransposeGradInferMeta(const MetaTensor& x, const MetaTensor& filter, const MetaTensor& dout, @@ -173,6 +178,11 @@ void NllLossGradInferMeta(const MetaTensor& input, MetaTensor* intput_grad, MetaConfig config = MetaConfig()); +void PixelUnshuffleGradInferMeta(const MetaTensor& out_grad, + int downscale_factor, + const std::string& data_format, + MetaTensor* x_grad); + void PsroiPoolGradInferMeta(const MetaTensor& x, const MetaTensor& rois, paddle::optional rois_num, diff --git a/paddle/phi/infermeta/unary.cc b/paddle/phi/infermeta/unary.cc index e3e1211e3ece8..cff14308c7fe9 100644 --- a/paddle/phi/infermeta/unary.cc +++ b/paddle/phi/infermeta/unary.cc @@ -228,13 +228,6 @@ void CholeskyInferMeta(const MetaTensor& x, bool upper, MetaTensor* out) { out->set_dtype(x.dtype()); } -void CopyToInferMeta(const MetaTensor& x, - Backend backend, - bool blocking, - MetaTensor* out) { - UnchangedInferMeta(x, out); -} - void CreateLikeInferMeta(const MetaTensor& x, DataType dtype, MetaTensor* out) { out->set_dims(x.dims()); out->set_dtype(dtype == DataType::UNDEFINED ? x.dtype() : dtype); @@ -1423,6 +1416,66 @@ void PixelShuffleGradInferMeta(const MetaTensor& out_grad, x_grad->set_dtype(out_grad.dtype()); } +void PixelUnshuffleInferMeta(const MetaTensor& x, + int downscale_factor, + const std::string& data_format, + MetaTensor* out) { + auto input_dims = x.dims(); + PADDLE_ENFORCE_EQ(input_dims.size(), + 4, + phi::errors::InvalidArgument( + "Input should be a 4-D tensor of format [N, C, H, W] " + "or [N, H, W, C], but got %u.", + input_dims.size())); + PADDLE_ENFORCE_GE(downscale_factor, + 1, + phi::errors::InvalidArgument( + "downscale_factor should be larger than 0.")); + PADDLE_ENFORCE_EQ(data_format == "NCHW" || data_format == "NHWC", + true, + phi::errors::InvalidArgument( + "data_format must be one of " + "NCHW and NHWC. But recevied data_format: %s", + data_format)); + + const bool channel_last = (data_format == "NHWC"); + + if (!channel_last) { + PADDLE_ENFORCE_EQ( + (input_dims[2] % downscale_factor) == 0 && + (input_dims[3] % downscale_factor) == 0, + true, + phi::errors::InvalidArgument("Downscale factor[%u] should divide both " + "height[%u] and width[%u]", + downscale_factor, + input_dims[2], + input_dims[3])); + } else { + PADDLE_ENFORCE_EQ( + (input_dims[1] % downscale_factor) == 0 && + (input_dims[2] % downscale_factor) == 0, + true, + phi::errors::InvalidArgument("Downscale factor[%u] should divide both " + "height[%u] and width[%u]", + downscale_factor, + input_dims[1], + input_dims[2])); + } + auto output_dims = input_dims; + output_dims[0] = input_dims[0]; + if (!channel_last) { + output_dims[1] = input_dims[1] * (downscale_factor * downscale_factor); + output_dims[2] = input_dims[2] / downscale_factor; + output_dims[3] = input_dims[3] / downscale_factor; + } else { + output_dims[1] = input_dims[1] / downscale_factor; + output_dims[2] = input_dims[2] / downscale_factor; + output_dims[3] = input_dims[3] * (downscale_factor * downscale_factor); + } + out->set_dtype(x.dtype()); + out->set_dims(output_dims); +} + void PNormInferMeta(const MetaTensor& x, float porder, int axis, @@ -2267,8 +2320,7 @@ void SumRawInferMeta(const MetaTensor& x, if (dtype != DataType::UNDEFINED) { out_dtype = dtype; } else { - if (x.dtype() == DataType::BOOL || x.dtype() == DataType::INT32 || - x.dtype() == DataType::INT64) { + if (x.dtype() == DataType::BOOL || x.dtype() == DataType::INT32) { out_dtype = DataType::INT64; } else { out_dtype = x.dtype(); @@ -3006,8 +3058,53 @@ void WhereIndexInferMeta(const MetaTensor& condition, MetaTensor* out) { out->set_dtype(DataType::INT64); } +void ChannelShuffleInferMeta(const MetaTensor& x, + int groups, + const std::string& data_format, + MetaTensor* out) { + auto input_dims = x.dims(); + PADDLE_ENFORCE_EQ(input_dims.size(), + 4, + phi::errors::InvalidArgument( + "Input should be a 4-D tensor of format [N, C, H, W] " + "or [N, H, W, C], but got %u.", + input_dims.size())); + PADDLE_ENFORCE_GE( + groups, + 1, + phi::errors::InvalidArgument("groups should be larger than 0.")); + PADDLE_ENFORCE_EQ(data_format == "NCHW" || data_format == "NHWC", + true, + phi::errors::InvalidArgument( + "data_format must be one of " + "NCHW and NHWC. But recevied data_format: %s", + data_format)); + + const bool channel_last = (data_format == "NHWC"); + + if (!channel_last) { + PADDLE_ENFORCE_EQ(input_dims[1] % groups, + 0, + phi::errors::InvalidArgument( + "The number of groups to divide channels in [%u] " + "should divide the number of channel [%u]", + groups, + input_dims[1])); + } else { + PADDLE_ENFORCE_EQ(input_dims[3] % groups, + 0, + phi::errors::InvalidArgument( + "The number of groups to divide channels in [%u] " + "should divide the number of channel [%u]", + groups, + input_dims[3])); + } + auto output_dims = input_dims; + out->set_dtype(x.dtype()); + out->set_dims(output_dims); +} + } // namespace phi -PD_REGISTER_INFER_META_FN(copy_to, phi::CopyToInferMeta); PD_REGISTER_INFER_META_FN(flatten, phi::FlattenInferMeta); PD_REGISTER_INFER_META_FN(split, phi::SplitInferMeta); diff --git a/paddle/phi/infermeta/unary.h b/paddle/phi/infermeta/unary.h index ac5040388b334..eef750b852f06 100644 --- a/paddle/phi/infermeta/unary.h +++ b/paddle/phi/infermeta/unary.h @@ -58,11 +58,6 @@ void CastInferMeta(const MetaTensor& x, DataType out_dtype, MetaTensor* out); void CholeskyInferMeta(const MetaTensor& x, bool upper, MetaTensor* out); -void CopyToInferMeta(const MetaTensor& x, - Backend backend, - bool blocking, - MetaTensor* out); - void CreateLikeInferMeta(const MetaTensor& x, DataType dtype, MetaTensor* out); void CumsumInferMeta(const MetaTensor& x, @@ -209,6 +204,11 @@ void PixelShuffleGradInferMeta(const MetaTensor& out_grad, const std::string& data_format, MetaTensor* x_grad); +void PixelUnshuffleInferMeta(const MetaTensor& x, + int downscale_factor, + const std::string& data_format, + MetaTensor* out); + void PNormInferMeta(const MetaTensor& x, float porder, int axis, @@ -440,4 +440,9 @@ void OneHotInferMeta(const MetaTensor& x, const Scalar& depth, MetaTensor* out); void WhereIndexInferMeta(const MetaTensor& condition, MetaTensor* out); +void ChannelShuffleInferMeta(const MetaTensor& x, + int groups, + const std::string& data_format, + MetaTensor* out); + } // namespace phi diff --git a/paddle/phi/kernels/CMakeLists.txt b/paddle/phi/kernels/CMakeLists.txt index a3a71ab692245..437c55c840f1a 100644 --- a/paddle/phi/kernels/CMakeLists.txt +++ b/paddle/phi/kernels/CMakeLists.txt @@ -36,7 +36,7 @@ set(MANUAL_BUILD_KERNELS ${AUTOTUNE_KERNELS} cross_entropy_kernel adam_kernel ad matrix_power_kernel matrix_power_grad_kernel maxout_kernel maxout_grad_kernel pool_kernel put_along_axis_kernel put_along_axis_grad_kernel segment_pool_kernel segment_pool_grad_kernel softmax_kernel softmax_grad_kernel take_along_axis_kernel take_along_axis_grad_kernel - triangular_solve_grad_kernel determinant_grad_kernel reduce_sum_kernel rnn_kernel rnn_grad_kernel warpctc_kernel warpctc_grad_kernel) + triangular_solve_grad_kernel determinant_grad_kernel reduce_sum_kernel reduce_mean_kernel rnn_kernel rnn_grad_kernel warpctc_kernel warpctc_grad_kernel) foreach(src ${AUTOTUNE_KERNELS}) kernel_library(${src} DEPS ${COMMON_KERNEL_DEPS} switch_autotune) endforeach() @@ -52,6 +52,7 @@ kernel_library(hierarchical_sigmoid_grad_kernel DEPS ${COMMON_KERNEL_DEPS} matri kernel_library(gumbel_softmax_kernel DEPS ${COMMON_KERNEL_DEPS} softmax) kernel_library(gumbel_softmax_grad_kernel DEPS ${COMMON_KERNEL_DEPS} softmax) kernel_library(reduce_sum_kernel DEPS ${COMMON_KERNEL_DEPS} cast_kernel) +kernel_library(reduce_mean_kernel DEPS ${COMMON_KERNEL_DEPS} cast_kernel) kernel_library(matrix_power_kernel DEPS ${COMMON_KERNEL_DEPS} matrix_inverse) kernel_library(matrix_power_grad_kernel DEPS ${COMMON_KERNEL_DEPS} matrix_inverse) kernel_library(maxout_kernel DEPS ${COMMON_KERNEL_DEPS} maxouting) diff --git a/paddle/phi/kernels/channel_shuffle_grad_kernel.h b/paddle/phi/kernels/channel_shuffle_grad_kernel.h new file mode 100644 index 0000000000000..ac89f3336bc76 --- /dev/null +++ b/paddle/phi/kernels/channel_shuffle_grad_kernel.h @@ -0,0 +1,29 @@ +// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#include +#include "paddle/phi/core/dense_tensor.h" + +namespace phi { + +template +void ChannelShuffleGradKernel(const Context& dev_ctx, + const DenseTensor& out_grad, + int groups, + const std::string& data_format, + DenseTensor* x_grad); + +} // namespace phi diff --git a/paddle/phi/kernels/channel_shuffle_kernel.h b/paddle/phi/kernels/channel_shuffle_kernel.h new file mode 100644 index 0000000000000..12de25606dd96 --- /dev/null +++ b/paddle/phi/kernels/channel_shuffle_kernel.h @@ -0,0 +1,29 @@ +// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#include +#include "paddle/phi/core/dense_tensor.h" + +namespace phi { + +template +void ChannelShuffleKernel(const Context& dev_ctx, + const DenseTensor& x, + int groups, + const std::string& data_format, + DenseTensor* out); + +} // namespace phi diff --git a/paddle/phi/kernels/cpu/channel_shuffle_grad_kernel.cc b/paddle/phi/kernels/cpu/channel_shuffle_grad_kernel.cc new file mode 100644 index 0000000000000..fcc91b2191673 --- /dev/null +++ b/paddle/phi/kernels/cpu/channel_shuffle_grad_kernel.cc @@ -0,0 +1,26 @@ +// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "paddle/phi/kernels/channel_shuffle_grad_kernel.h" +#include "paddle/phi/kernels/impl/channel_shuffle_grad_kernel_impl.h" + +#include "paddle/phi/backends/cpu/cpu_context.h" +#include "paddle/phi/core/kernel_registry.h" + +PD_REGISTER_KERNEL(channel_shuffle_grad, + CPU, + ALL_LAYOUT, + phi::ChannelShuffleGradKernel, + float, + double) {} diff --git a/paddle/phi/kernels/cpu/channel_shuffle_kernel.cc b/paddle/phi/kernels/cpu/channel_shuffle_kernel.cc new file mode 100644 index 0000000000000..95d19ec6a7746 --- /dev/null +++ b/paddle/phi/kernels/cpu/channel_shuffle_kernel.cc @@ -0,0 +1,26 @@ +// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "paddle/phi/kernels/channel_shuffle_kernel.h" +#include "paddle/phi/kernels/impl/channel_shuffle_kernel_impl.h" + +#include "paddle/phi/backends/cpu/cpu_context.h" +#include "paddle/phi/core/kernel_registry.h" + +PD_REGISTER_KERNEL(channel_shuffle, + CPU, + ALL_LAYOUT, + phi::ChannelShuffleKernel, + float, + double) {} diff --git a/paddle/phi/kernels/cpu/einsum_grad_kernel.cc b/paddle/phi/kernels/cpu/einsum_grad_kernel.cc new file mode 100644 index 0000000000000..2cfc2f92204fc --- /dev/null +++ b/paddle/phi/kernels/cpu/einsum_grad_kernel.cc @@ -0,0 +1,22 @@ +// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "paddle/phi/kernels/einsum_grad_kernel.h" + +#include "paddle/phi/backends/cpu/cpu_context.h" +#include "paddle/phi/core/kernel_registry.h" +#include "paddle/phi/kernels/impl/einsum_grad_impl.h" + +PD_REGISTER_KERNEL( + einsum_grad, CPU, ALL_LAYOUT, phi::EinsumGradKernel, float, double) {} diff --git a/paddle/phi/kernels/cpu/einsum_kernel.cc b/paddle/phi/kernels/cpu/einsum_kernel.cc new file mode 100644 index 0000000000000..3e25a65526d89 --- /dev/null +++ b/paddle/phi/kernels/cpu/einsum_kernel.cc @@ -0,0 +1,20 @@ +// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "paddle/phi/kernels/einsum_kernel.h" +#include "paddle/phi/backends/cpu/cpu_context.h" +#include "paddle/phi/core/kernel_registry.h" +#include "paddle/phi/kernels/impl/einsum_impl.h" + +PD_REGISTER_KERNEL(einsum, CPU, ALL_LAYOUT, phi::EinsumKernel, float, double) {} diff --git a/paddle/phi/kernels/cpu/pixel_unshuffle_grad_kernel.cc b/paddle/phi/kernels/cpu/pixel_unshuffle_grad_kernel.cc new file mode 100644 index 0000000000000..ef61fca35957e --- /dev/null +++ b/paddle/phi/kernels/cpu/pixel_unshuffle_grad_kernel.cc @@ -0,0 +1,26 @@ +// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "paddle/phi/kernels/pixel_unshuffle_grad_kernel.h" +#include "paddle/phi/kernels/impl/pixel_unshuffle_grad_kernel_impl.h" + +#include "paddle/phi/backends/cpu/cpu_context.h" +#include "paddle/phi/core/kernel_registry.h" + +PD_REGISTER_KERNEL(pixel_unshuffle_grad, + CPU, + ALL_LAYOUT, + phi::PixelUnshuffleGradKernel, + float, + double) {} diff --git a/paddle/phi/kernels/cpu/pixel_unshuffle_kernel.cc b/paddle/phi/kernels/cpu/pixel_unshuffle_kernel.cc new file mode 100644 index 0000000000000..9f4bc747f3209 --- /dev/null +++ b/paddle/phi/kernels/cpu/pixel_unshuffle_kernel.cc @@ -0,0 +1,26 @@ +// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "paddle/phi/kernels/pixel_unshuffle_kernel.h" +#include "paddle/phi/kernels/impl/pixel_unshuffle_kernel_impl.h" + +#include "paddle/phi/backends/cpu/cpu_context.h" +#include "paddle/phi/core/kernel_registry.h" + +PD_REGISTER_KERNEL(pixel_unshuffle, + CPU, + ALL_LAYOUT, + phi::PixelUnshuffleKernel, + float, + double) {} diff --git a/paddle/phi/kernels/cpu/reduce_sum_kernel.cc b/paddle/phi/kernels/cpu/reduce_sum_kernel.cc index 32b12ea684528..0b4c4b9f4705a 100644 --- a/paddle/phi/kernels/cpu/reduce_sum_kernel.cc +++ b/paddle/phi/kernels/cpu/reduce_sum_kernel.cc @@ -29,6 +29,9 @@ void SumRawKernel(const Context& dev_ctx, bool reduce_all, DataType out_dtype, DenseTensor* out) { + if (out_dtype == DataType::UNDEFINED && out->dtype() != x.dtype()) { + out_dtype = out->dtype(); + } phi::Reduce( dev_ctx, x, reduce_all, dims, keep_dim, out_dtype, out); } diff --git a/paddle/phi/kernels/cpu/where_grad_kernel.cc b/paddle/phi/kernels/cpu/where_grad_kernel.cc index 67c8cee1038c7..a9cdbd7ad77cc 100644 --- a/paddle/phi/kernels/cpu/where_grad_kernel.cc +++ b/paddle/phi/kernels/cpu/where_grad_kernel.cc @@ -14,6 +14,8 @@ #include "paddle/phi/kernels/where_grad_kernel.h" +#include "paddle/phi/core/kernel_registry.h" + namespace phi { template diff --git a/paddle/phi/kernels/cpu/where_kernel.cc b/paddle/phi/kernels/cpu/where_kernel.cc index f624c13c26229..353d11c93c1cc 100644 --- a/paddle/phi/kernels/cpu/where_kernel.cc +++ b/paddle/phi/kernels/cpu/where_kernel.cc @@ -14,6 +14,8 @@ #include "paddle/phi/kernels/where_kernel.h" +#include "paddle/phi/core/kernel_registry.h" + namespace phi { template diff --git a/paddle/phi/kernels/einsum_grad_kernel.h b/paddle/phi/kernels/einsum_grad_kernel.h new file mode 100644 index 0000000000000..5c1970e775825 --- /dev/null +++ b/paddle/phi/kernels/einsum_grad_kernel.h @@ -0,0 +1,28 @@ +// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#include "paddle/phi/core/dense_tensor.h" + +namespace phi { + +template +void EinsumGradKernel(const Context& dev_ctx, + const std::vector& x, + const DenseTensor& out_grad, + const std::string& equation, + std::vector x_grad); + +} // namespace phi diff --git a/paddle/phi/kernels/einsum_kernel.h b/paddle/phi/kernels/einsum_kernel.h new file mode 100644 index 0000000000000..3d9e8feda748d --- /dev/null +++ b/paddle/phi/kernels/einsum_kernel.h @@ -0,0 +1,27 @@ +// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#include "paddle/phi/core/dense_tensor.h" + +namespace phi { + +template +void EinsumKernel(const Context& dev_ctx, + const std::vector& inputs, + const std::string& equation, + DenseTensor* out); + +} // namespace phi diff --git a/paddle/phi/kernels/funcs/activation_functor.h b/paddle/phi/kernels/funcs/activation_functor.h index 84da69ed5da02..b75477a1af982 100644 --- a/paddle/phi/kernels/funcs/activation_functor.h +++ b/paddle/phi/kernels/funcs/activation_functor.h @@ -13,6 +13,7 @@ // limitations under the License. #pragma once + #include #include #include @@ -33,7 +34,6 @@ #include "paddle/phi/common/float16.h" #include "paddle/phi/core/dense_tensor.h" #include "paddle/phi/core/enforce.h" -#include "paddle/phi/core/kernel_registry.h" #include "paddle/phi/kernels/funcs/eigen/common.h" #include "paddle/phi/kernels/funcs/eigen/extensions.h" diff --git a/paddle/phi/kernels/funcs/broadcast_function.h b/paddle/phi/kernels/funcs/broadcast_function.h index 7634c2462738b..aafa40a3d01bf 100644 --- a/paddle/phi/kernels/funcs/broadcast_function.h +++ b/paddle/phi/kernels/funcs/broadcast_function.h @@ -31,13 +31,14 @@ struct DimensionsTransform { using DimVector = std::vector; typedef void (*MergeFunctor)( bool &, std::vector &, DimVector &, int, int); + int64_t N; int64_t dim_size; DimVector out_dims; std::vector in_dims; private: - // To compensate the lackage of input_tensors` dimension with input variable - // 'axis' + // To compensate the lackage of input_tensors` dimension with input + // variable 'axis'. void InputDimensionsExtend(int N, int axis) { for (auto &in_dim : in_dims) { int64_t in_idx = 0; @@ -82,6 +83,8 @@ struct DimensionsTransform { std::reverse(out_dims.begin(), out_dims.end()); } + // Merge sequential dimension to shrink calculation cost for + // offset computation in CUDA Kernel. template __inline__ void MergeDimensions(MergeFunctor merge_func, int N) { auto VectorReorganise = [](DimVector *vec, int l_idx, int m_idx) { @@ -120,11 +123,44 @@ struct DimensionsTransform { } } + // To judge whether shape of any input tensors is sequential + // 1-value-dimensions, and metric the length of it. + int GetSequentialOneDimLength(int *swap_index) { + int index = 0; + int max_one_length = 0; + for (int j = 0; j < N; ++j) { + int seq_one_length = 0; + bool active_seq = false; + + for (int i = 0; i < dim_size; ++i) { + if (!active_seq && in_dims[j][i] == 1) { + seq_one_length = 1; + active_seq = true; + } else if (active_seq) { + if (in_dims[j][i] == 1) { + seq_one_length++; + } else { + active_seq = false; + } + } + } + max_one_length = + seq_one_length > max_one_length ? seq_one_length : max_one_length; + index = seq_one_length > max_one_length ? j : index; + } + + if (max_one_length > 1) { + std::swap(in_dims[0], in_dims[index]); + *swap_index = index; + } + return max_one_length; + } + public: explicit DimensionsTransform(const std::vector &ins, const phi::DDim &dims, int axis) { - const int N = std::max(static_cast(ins.size()), 2); + N = std::max(static_cast(ins.size()), 2); dim_size = dims.size(); out_dims = phi::vectorize(dims); in_dims.resize(N); @@ -140,6 +176,11 @@ struct DimensionsTransform { } InputDimensionsExtend(N, axis); + // To Merge the dimensions of input_tensors while the consequtive + // equal-dimensions appears. Example below : + // in_1.shape = [2, 3, 4, 5] in_1.shape = [2, 12, 5] + // in_2.shape = [1, 3, 4, 5] -> in_2.shape = [1, 12, 5] + // in_3.shape = [2, 3, 4, 1] in_3.shape = [2, 12, 1] auto merge_sequential_dims = [](bool &equal, std::vector &in_dims, DimVector &out, @@ -149,6 +190,17 @@ struct DimensionsTransform { equal &= (in_dims[0][i] == in_dims[j][i]) ? true : false; } }; + MergeFunctor merge_ptr = merge_sequential_dims; + MergeDimensions(merge_ptr, N); + + // To Merge the dimension of input_tensors while the sequential + // 1-value-dimensions appears. Example below : + // in_1.shape = [2, 1, 1, 5] in_1.shape = [2, 1, 5] + // in_2.shape = [2, 3, 4, 5] -> in_2.shape = [1, 12, 5] + // in_3.shape = [2, 3, 4, 1] in_3.shape = [2, 12, 1] + // Caution: Once 1-value-dimensions appears, the corresponding + // shape position of other input tensors must be same with the + // output tensor`s shape, or incorrect merge may occur. auto merge_sequential_one_dims = [](bool &equal, std::vector &in_dims, DimVector &out, @@ -161,27 +213,13 @@ struct DimensionsTransform { } } }; - // To Merge the dimensions of input_tensors while the consequtive - // equal-dimensions appears. - MergeFunctor merge_ptr = merge_sequential_dims; - MergeDimensions(merge_ptr, N); - - int min_idx = 0; - int min_val = std::accumulate( - in_dims[0].begin(), in_dims[0].end(), 1, std::multiplies()); - for (int j = 1; j < N; ++j) { - int temp = std::accumulate( - in_dims[j].begin(), in_dims[j].end(), 1, std::multiplies()); - min_val = min_val > temp ? temp : min_val; - min_idx = min_val == temp ? j : min_idx; + int swap_idx = 0; + int max_one_length = GetSequentialOneDimLength(&swap_idx); + if (max_one_length > 1) { + merge_ptr = merge_sequential_one_dims; + MergeDimensions(merge_ptr, N); + std::swap(in_dims[swap_idx], in_dims[0]); } - std::swap(in_dims[0], in_dims[min_idx]); - - // To Merge the dimension of input_tensors while the consequtive - // 1-value-dimensions appears. - merge_ptr = merge_sequential_one_dims; - MergeDimensions(merge_ptr, N); - std::swap(in_dims[min_idx], in_dims[0]); } }; @@ -554,6 +592,7 @@ void BroadcastKernel(const KPDevice &ctx, int axis, Functor func) { std::vector dims_size; + dims_size.reserve(ins.size()); bool no_broadcast_flag = true; for (auto *in : ins) { no_broadcast_flag &= ins[0]->dims() == in->dims(); diff --git a/paddle/phi/kernels/funcs/elementwise_grad_base.h b/paddle/phi/kernels/funcs/elementwise_grad_base.h index 1021b510b26cd..7508d8ee8cdc8 100644 --- a/paddle/phi/kernels/funcs/elementwise_grad_base.h +++ b/paddle/phi/kernels/funcs/elementwise_grad_base.h @@ -15,6 +15,7 @@ limitations under the License. */ #pragma once #include "paddle/phi/backends/all_context.h" +#include "paddle/phi/backends/gpu/gpu_info.h" #include "paddle/phi/core/dense_tensor.h" #include "paddle/phi/kernels/funcs/common_shape.h" #include "paddle/phi/kernels/funcs/elementwise_utils.h" @@ -978,7 +979,7 @@ static void ElemwiseGradBroadcast1CUDA(gpuStream_t stream, // suppose perfoemance improves with h increased. dim3 block_size = dim3(BLOCK_X, BLOCK_Y); dim3 grid_size = dim3((w + BLOCK_X - 1) / BLOCK_X); - auto gplace = phi::GPUPlace(); + auto gplace = phi::GPUPlace(phi::backends::gpu::GetCurrentDeviceId()); auto *ctx = static_cast( paddle::platform::DeviceContextPool::Instance().Get(gplace)); paddle::platform::LimitGridDim(*ctx, &grid_size); @@ -1003,7 +1004,7 @@ static void ElemwiseGradBroadcast2CUDA(gpuStream_t stream, T *dy) { int block_size = std::min(ELEMWISE_MAX_BLOCK_DIM, pre * post); dim3 grid_size = dim3(n); - auto gplace = phi::GPUPlace(); + auto gplace = phi::GPUPlace(phi::backends::gpu::GetCurrentDeviceId()); auto *ctx = static_cast( paddle::platform::DeviceContextPool::Instance().Get(gplace)); paddle::platform::LimitGridDim(*ctx, &grid_size); diff --git a/paddle/phi/kernels/gpu/channel_shuffle_grad_kernel.cu b/paddle/phi/kernels/gpu/channel_shuffle_grad_kernel.cu new file mode 100644 index 0000000000000..63d3d4a554f81 --- /dev/null +++ b/paddle/phi/kernels/gpu/channel_shuffle_grad_kernel.cu @@ -0,0 +1,26 @@ +// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "paddle/phi/kernels/channel_shuffle_grad_kernel.h" +#include "paddle/phi/kernels/impl/channel_shuffle_grad_kernel_impl.h" + +#include "paddle/phi/backends/gpu/gpu_context.h" +#include "paddle/phi/core/kernel_registry.h" + +PD_REGISTER_KERNEL(channel_shuffle_grad, + GPU, + ALL_LAYOUT, + phi::ChannelShuffleGradKernel, + float, + double) {} diff --git a/paddle/phi/kernels/gpu/channel_shuffle_kernel.cu b/paddle/phi/kernels/gpu/channel_shuffle_kernel.cu new file mode 100644 index 0000000000000..f85cb4aafd1dc --- /dev/null +++ b/paddle/phi/kernels/gpu/channel_shuffle_kernel.cu @@ -0,0 +1,26 @@ +// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "paddle/phi/kernels/channel_shuffle_kernel.h" +#include "paddle/phi/kernels/impl/channel_shuffle_kernel_impl.h" + +#include "paddle/phi/backends/gpu/gpu_context.h" +#include "paddle/phi/core/kernel_registry.h" + +PD_REGISTER_KERNEL(channel_shuffle, + GPU, + ALL_LAYOUT, + phi::ChannelShuffleKernel, + float, + double) {} diff --git a/paddle/phi/kernels/gpu/cumsum_kernel.cu b/paddle/phi/kernels/gpu/cumsum_kernel.cu index e04f2b5f87658..13975ddd3ef89 100644 --- a/paddle/phi/kernels/gpu/cumsum_kernel.cu +++ b/paddle/phi/kernels/gpu/cumsum_kernel.cu @@ -39,14 +39,12 @@ __device__ void BlockReverse( int tx = threadIdx.x; int offset = tx; - int in_index = src_base + offset; - if (offset >= valid_item) { - sh_mem[offset] = 0; - } else { - int sh_mem_index = BLOCK_SIZE - offset - 1; - T data = idata[in_index]; - sh_mem[sh_mem_index] = data; + T src_data = 0; + int src_offset = BLOCK_SIZE - offset - 1; + if (src_offset < valid_item) { + src_data = idata[src_base + src_offset]; } + sh_mem[offset] = src_data; __syncthreads(); int out_index = dst_base - offset; diff --git a/paddle/phi/kernels/gpu/einsum_grad_kernel.cu b/paddle/phi/kernels/gpu/einsum_grad_kernel.cu new file mode 100644 index 0000000000000..c8a8745f34522 --- /dev/null +++ b/paddle/phi/kernels/gpu/einsum_grad_kernel.cu @@ -0,0 +1,22 @@ +// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "paddle/phi/kernels/einsum_kernel.h" + +#include "paddle/phi/backends/gpu/gpu_context.h" +#include "paddle/phi/core/kernel_registry.h" +#include "paddle/phi/kernels/impl/einsum_grad_impl.h" + +PD_REGISTER_KERNEL( + einsum_grad, GPU, ALL_LAYOUT, phi::EinsumGradKernel, float, double) {} diff --git a/paddle/phi/kernels/gpu/einsum_kernel.cu b/paddle/phi/kernels/gpu/einsum_kernel.cu new file mode 100644 index 0000000000000..d73e154eb40f7 --- /dev/null +++ b/paddle/phi/kernels/gpu/einsum_kernel.cu @@ -0,0 +1,21 @@ +// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "paddle/phi/kernels/einsum_kernel.h" + +#include "paddle/phi/backends/gpu/gpu_context.h" +#include "paddle/phi/core/kernel_registry.h" +#include "paddle/phi/kernels/impl/einsum_impl.h" + +PD_REGISTER_KERNEL(einsum, GPU, ALL_LAYOUT, phi::EinsumKernel, float, double) {} diff --git a/paddle/phi/kernels/gpu/multinomial_kernel.cu b/paddle/phi/kernels/gpu/multinomial_kernel.cu index ef6cd1323a9df..21a506a840cc7 100644 --- a/paddle/phi/kernels/gpu/multinomial_kernel.cu +++ b/paddle/phi/kernels/gpu/multinomial_kernel.cu @@ -133,11 +133,10 @@ void MultinomialKernel(const Context& dev_ctx, DenseTensor* out) { auto* in_data = x.data(); int64_t* out_data = dev_ctx.template Alloc(out); - auto in_dims = x.dims(); - int64_t in_rank = in_dims.size(); - const int64_t num_categories = in_dims[in_rank - 1]; - const int64_t num_distributions = in_rank > 1 ? in_dims[in_rank - 2] : 1; + int64_t dim_size = in_dims.size(); + const int64_t num_categories = in_dims[dim_size - 1]; + const int64_t num_distributions = dim_size > 1 ? in_dims[dim_size - 2] : 1; // If replacement is False, it's not a replaceable sample. Every category // can be used only once. @@ -145,8 +144,8 @@ void MultinomialKernel(const Context& dev_ctx, int64_t in_data_numel = x.numel(); int64_t out_data_numel = out->numel(); + // Just use to PADDLE_ENFORCE error message T* cpu_in_data = new T[in_data_numel]; - int64_t* cpu_out_data = new int64_t[out_data_numel]; #ifdef PADDLE_WITH_HIP hipMemcpy( @@ -160,7 +159,7 @@ void MultinomialKernel(const Context& dev_ctx, for (size_t i = 0; i < num_distributions; ++i) { int zero_num = 0; for (size_t j = 0; j < num_categories; ++j) { - T weight = cpu_in_data[i * num_distributions + j]; + T weight = cpu_in_data[i * num_categories + j]; PADDLE_ENFORCE_GE( weight, 0, diff --git a/paddle/phi/kernels/gpu/pixel_unshuffle_grad_kernel.cu b/paddle/phi/kernels/gpu/pixel_unshuffle_grad_kernel.cu new file mode 100644 index 0000000000000..9cbbc5072aa25 --- /dev/null +++ b/paddle/phi/kernels/gpu/pixel_unshuffle_grad_kernel.cu @@ -0,0 +1,26 @@ +// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "paddle/phi/kernels/impl/pixel_unshuffle_grad_kernel_impl.h" +#include "paddle/phi/kernels/pixel_unshuffle_grad_kernel.h" + +#include "paddle/phi/backends/gpu/gpu_context.h" +#include "paddle/phi/core/kernel_registry.h" + +PD_REGISTER_KERNEL(pixel_unshuffle_grad, + GPU, + ALL_LAYOUT, + phi::PixelUnshuffleGradKernel, + float, + double) {} diff --git a/paddle/phi/kernels/gpu/pixel_unshuffle_kernel.cu b/paddle/phi/kernels/gpu/pixel_unshuffle_kernel.cu new file mode 100644 index 0000000000000..ca2e520ffde10 --- /dev/null +++ b/paddle/phi/kernels/gpu/pixel_unshuffle_kernel.cu @@ -0,0 +1,26 @@ +// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "paddle/phi/kernels/impl/pixel_unshuffle_kernel_impl.h" +#include "paddle/phi/kernels/pixel_unshuffle_kernel.h" + +#include "paddle/phi/backends/gpu/gpu_context.h" +#include "paddle/phi/core/kernel_registry.h" + +PD_REGISTER_KERNEL(pixel_unshuffle, + GPU, + ALL_LAYOUT, + phi::PixelUnshuffleKernel, + float, + double) {} diff --git a/paddle/phi/kernels/gpu/randperm_kernel.cu b/paddle/phi/kernels/gpu/randperm_kernel.cu index 4e488ed470df9..94f063512c06f 100644 --- a/paddle/phi/kernels/gpu/randperm_kernel.cu +++ b/paddle/phi/kernels/gpu/randperm_kernel.cu @@ -36,26 +36,29 @@ DECLARE_bool(use_curand); namespace phi { -template -__global__ void SwapRepeatKernel( - int* key, T* data, int n, uint64_t seed, uint64_t offset) { +template +__global__ void SwapRepeatKernel(keyT* key_out_data, + dataT* out_data, + int n, + uint64_t seed, + uint64_t offset) { size_t idx = static_cast(blockIdx.x * blockDim.x + threadIdx.x); - if (idx < n) return; + if (idx >= n - 1) return; // out of range - bool first_repeat = false; - if (data[idx] == data[idx + 1]) { + bool is_first_repeat = false; + if (key_out_data[idx] == key_out_data[idx + 1]) { if (idx == 0) { - first_repeat = true; - } else if (data[idx] != data[idx - 1]) { - first_repeat = true; + is_first_repeat = true; + } else if (key_out_data[idx] != key_out_data[idx - 1]) { + is_first_repeat = true; } } - if (!first_repeat) return; + if (!is_first_repeat) return; int repeat_size = 1; for (int i = idx; i < n; ++i) { - if (data[i] == data[i + 1]) { + if (key_out_data[i] == key_out_data[i + 1]) { ++repeat_size; } else { break; @@ -74,9 +77,9 @@ __global__ void SwapRepeatKernel( uint32_t r = hiprand(&state) % (i + 1); #endif if (r != i) { - T tmp = data[idx + i]; - data[idx + i] = data[idx + r]; - data[idx + r] = tmp; + dataT tmp = out_data[idx + i]; + out_data[idx + i] = out_data[idx + r]; + out_data[idx + r] = tmp; } } } @@ -138,10 +141,10 @@ void RandpermRawKernel( auto seed_offset = gen_cuda->IncrementOffset(n); auto config = phi::backends::gpu::GetGpuLaunchConfig1D(dev_ctx, n); - SwapRepeatKernel<<>>( + SwapRepeatKernel<<>>( key_out.data(), out_data, n, seed_offset.first, seed_offset.second); } diff --git a/paddle/phi/kernels/gpu/where_grad_kernel.cu b/paddle/phi/kernels/gpu/where_grad_kernel.cu index f21aca80e21b3..14cc1d311321d 100644 --- a/paddle/phi/kernels/gpu/where_grad_kernel.cu +++ b/paddle/phi/kernels/gpu/where_grad_kernel.cu @@ -14,6 +14,9 @@ #include "paddle/phi/kernels/where_grad_kernel.h" +#include "paddle/phi/backends/gpu/gpu_launch_config.h" +#include "paddle/phi/core/kernel_registry.h" + namespace phi { template diff --git a/paddle/phi/kernels/gpu/where_kernel.cu b/paddle/phi/kernels/gpu/where_kernel.cu index 03c24eea3a95a..a0be388065f4b 100644 --- a/paddle/phi/kernels/gpu/where_kernel.cu +++ b/paddle/phi/kernels/gpu/where_kernel.cu @@ -14,6 +14,8 @@ #include "paddle/phi/kernels/where_kernel.h" +#include "paddle/phi/backends/gpu/gpu_launch_config.h" +#include "paddle/phi/core/kernel_registry.h" #include "paddle/phi/kernels/funcs/broadcast_function.h" #include "paddle/phi/kernels/funcs/elementwise_functor.h" diff --git a/paddle/phi/kernels/impl/channel_shuffle_grad_kernel_impl.h b/paddle/phi/kernels/impl/channel_shuffle_grad_kernel_impl.h new file mode 100644 index 0000000000000..26bee763eca52 --- /dev/null +++ b/paddle/phi/kernels/impl/channel_shuffle_grad_kernel_impl.h @@ -0,0 +1,58 @@ +// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#include +#include + +#include "paddle/phi/core/dense_tensor.h" +#include "paddle/phi/kernels/funcs/math_function.h" + +namespace phi { + +template +void ChannelShuffleGradKernel(const Context& dev_ctx, + const DenseTensor& out_grad, + int groups, + const std::string& data_format, + DenseTensor* x_grad) { + auto* dout = &out_grad; + auto* dx = x_grad; + dev_ctx.template Alloc(dx); + bool channel_last = (data_format == "NHWC"); + auto do_dims = dout->dims(); + auto dx_dims = dx->dims(); + + DenseTensor t(*dout); + if (!channel_last) { + t.Resize({do_dims[0], do_dims[1] / groups, groups, do_dims[2], do_dims[3]}); + } else { + t.Resize({do_dims[0], do_dims[1], do_dims[2], do_dims[3] / groups, groups}); + } + auto axis = !channel_last ? std::vector{0, 2, 1, 3, 4} + : std::vector{0, 1, 2, 4, 3}; + + DenseTensor o(*dx); + if (!channel_last) { + o.Resize({dx_dims[0], groups, dx_dims[1] / groups, dx_dims[2], dx_dims[3]}); + } else { + o.Resize({dx_dims[0], dx_dims[1], dx_dims[2], groups, dx_dims[3] / groups}); + } + phi::funcs::Transpose trans; + trans(dev_ctx, t, &o, axis); + dx->Resize(dx_dims); +} + +} // namespace phi diff --git a/paddle/phi/kernels/impl/channel_shuffle_kernel_impl.h b/paddle/phi/kernels/impl/channel_shuffle_kernel_impl.h new file mode 100644 index 0000000000000..c723cd3622af9 --- /dev/null +++ b/paddle/phi/kernels/impl/channel_shuffle_kernel_impl.h @@ -0,0 +1,57 @@ +// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#include +#include + +#include "paddle/phi/core/dense_tensor.h" +#include "paddle/phi/kernels/funcs/math_function.h" + +namespace phi { + +template +void ChannelShuffleKernel(const Context& dev_ctx, + const DenseTensor& x, + int groups, + const std::string& data_format, + DenseTensor* out) { + auto* in = &x; + dev_ctx.template Alloc(out); + bool channel_last = (data_format == "NHWC"); + auto in_dims = in->dims(); + auto o_dims = out->dims(); + + DenseTensor t(*in); + if (!channel_last) { + t.Resize({in_dims[0], groups, in_dims[1] / groups, in_dims[2], in_dims[3]}); + } else { + t.Resize({in_dims[0], in_dims[1], in_dims[2], groups, in_dims[3] / groups}); + } + auto axis = !channel_last ? std::vector{0, 2, 1, 3, 4} + : std::vector{0, 1, 2, 4, 3}; + + DenseTensor o(*out); + if (!channel_last) { + o.Resize({in_dims[0], in_dims[1] / groups, groups, in_dims[2], in_dims[3]}); + } else { + o.Resize({in_dims[0], in_dims[1], in_dims[2], in_dims[3] / groups, groups}); + } + phi::funcs::Transpose trans; + trans(dev_ctx, t, &o, axis); + out->Resize(o_dims); +} + +} // namespace phi diff --git a/paddle/phi/kernels/impl/einsum_grad_impl.h b/paddle/phi/kernels/impl/einsum_grad_impl.h new file mode 100644 index 0000000000000..bd0143379ce15 --- /dev/null +++ b/paddle/phi/kernels/impl/einsum_grad_impl.h @@ -0,0 +1,181 @@ +// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +#pragma once + +#include "paddle/phi/core/dense_tensor.h" +#include "paddle/phi/kernels/impl/einsum_impl.h" +#include "paddle/phi/kernels/tile_kernel.h" +#include "paddle/utils/string/string_helper.h" + +namespace phi { +template +DenseTensor PerformTileAndReduction(const Context& dev_ctx, + const LabelMap& label2type, + const LabelMap& label2shape, + const std::vector& broadcast_dims, + const std::vector& ellipsis_dims, + std::string op_label, // value pass + DenseTensor& t) { // NOLINT + ReplaceEllipsis(op_label); + DenseTensor ret; + std::vector repeat_times; + std::vector resize_dims; + std::vector recover_shape; + for (int c : op_label) { + if (label2type[c] == LabelType::Reduction) { + // '.' can't be Reduction, so we don't deal '.' here. + repeat_times.push_back(label2shape[c]); + resize_dims.push_back(1); + recover_shape.push_back(label2shape[c]); + } else { + if (c != '.') { + resize_dims.push_back(label2shape[c]); + repeat_times.push_back(1); + recover_shape.push_back(label2shape[c]); + } else { + int n_dims = broadcast_dims.size(); + resize_dims.insert( + resize_dims.end(), broadcast_dims.begin(), broadcast_dims.end()); + recover_shape.insert( + recover_shape.end(), ellipsis_dims.begin(), ellipsis_dims.end()); + while (n_dims--) repeat_times.push_back(1); + } + } + } + t.Resize(make_ddim(resize_dims)); + DenseTensor after_tile; + TileKernel(dev_ctx, t, repeat_times, &after_tile); + size_t n_ellipsis_idx = op_label.find(".", 0); + if (n_ellipsis_idx != std::string::npos) { + // may be we need reduce. broadcast_dims is not equal to ellipsis dims. + std::vector to_reduce; + for (size_t i = 0; i < broadcast_dims.size() - ellipsis_dims.size(); ++i) + to_reduce.push_back(i + n_ellipsis_idx); + + int new_offset = + n_ellipsis_idx + broadcast_dims.size() - ellipsis_dims.size(); + for (size_t i = 0; i < ellipsis_dims.size(); ++i) + if (ellipsis_dims[i] == 1) to_reduce.push_back(i + new_offset); + + VLOG(5) << "PermformTileAndReduction: reduce sum axis: " + << paddle::string::join_strings(to_reduce, ","); + if (to_reduce.size() != 0) { + ret = Sum(dev_ctx, + after_tile, + to_reduce, + after_tile.dtype(), + false); // not keep dim. + } else { + ret = after_tile; + } + } else { + ret = after_tile; + } + VLOG(5) << "PermformTileAndReduction: recover shape: " + << paddle::string::join_strings(recover_shape, ","); + ret.Resize(make_ddim(recover_shape)); + return ret; +} + +template +void EinsumGradKernel(const Context& dev_ctx, + const std::vector& x, + const DenseTensor& out_grad, + const std::string& equation, + std::vector x_grad) { + VLOG(5) << "Start EisumGradKernel:"; + LabelMap labelshape(0); + LabelMap labeltype(LabelType::Reduction); + std::vector label2perms(x.size(), LabelMap(-1)); + std::vector all_labels; // order: ABO, AO, BO, AB, Reduce + std::vector> ellipsis_dims(2); + std::vector broadcast_dims; + std::vector output_dims; + + std::vector input_dims; + for (auto& i : x) { + input_dims.push_back(i->dims()); + } + std::string right; + ParseEinsumEquation(equation, + input_dims, + &labelshape, + &labeltype, + &all_labels, + &label2perms, + &ellipsis_dims, + &broadcast_dims, + &output_dims, + &right); + + auto gather_labels_except_reduction = [&labeltype](std::string all) { + std::string res(""); + for (auto c : all) + if (labeltype[static_cast(c)] != LabelType::Reduction) res += c; + return res; + }; + if (x.size() == 1) { // Unary + auto splits = paddle::string::split_string(equation, "->"); + auto left = splits[0]; + right = splits[1].substr(1); + auto new_equation = right + "->" + gather_labels_except_reduction(left); + auto new_operands = std::vector(); + new_operands.push_back(&out_grad); + DenseTensor before_tile; + EinsumKernel(dev_ctx, new_operands, new_equation, &before_tile); + *(x_grad[0]) = PerformTileAndReduction(dev_ctx, + labeltype, + labelshape, + broadcast_dims, + ellipsis_dims[0], + left, + before_tile); + } else { + auto splits = paddle::string::split_string(equation, "->"); + auto left = splits[0]; + auto ops = paddle::string::split_string(left, ","); + right = splits[1].substr(1); + + auto equation_for_A = + right + "," + ops[1] + "->" + gather_labels_except_reduction(ops[0]); + auto equation_for_B = + right + "," + ops[0] + "->" + gather_labels_except_reduction(ops[1]); + auto operands_for_A = std::vector(); + auto operands_for_B = std::vector(); + DenseTensor dA, dB; + operands_for_A.push_back(&out_grad); + operands_for_A.push_back(x[1]); + operands_for_B.push_back(&out_grad); + operands_for_B.push_back(x[0]); + + DenseTensor before_tile; + EinsumKernel(dev_ctx, operands_for_A, equation_for_A, &dA); + EinsumKernel(dev_ctx, operands_for_B, equation_for_B, &dB); + *(x_grad[0]) = PerformTileAndReduction(dev_ctx, + labeltype, + labelshape, + broadcast_dims, + ellipsis_dims[0], + ops[0], + dA); + *(x_grad[1]) = PerformTileAndReduction(dev_ctx, + labeltype, + labelshape, + broadcast_dims, + ellipsis_dims[1], + ops[1], + dB); + } +} +} // namespace phi diff --git a/paddle/phi/kernels/impl/einsum_impl.h b/paddle/phi/kernels/impl/einsum_impl.h new file mode 100644 index 0000000000000..d4be007a07fc0 --- /dev/null +++ b/paddle/phi/kernels/impl/einsum_impl.h @@ -0,0 +1,586 @@ +// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +#pragma once + +#include "paddle/fluid/framework/infershape_utils.h" +#include "paddle/phi/core/dense_tensor.h" +#include "paddle/phi/kernels/matmul_kernel.h" +#include "paddle/phi/kernels/reduce_sum_kernel.h" +#include "paddle/phi/kernels/transpose_kernel.h" +#include "paddle/utils/string/string_helper.h" + +namespace phi { +// check the validation of the Einsum equation. +// 1. the label must between 'a' - 'z'. +// 2. the dim of the same label must be same. +// 3. the broad cast dims in two operands is broadcastable. +// 4. there must exist '->' and the default output is complete in python. +// may be we can skip validation check in C++ and just put it in python. +inline static void ValidationCheck(const std::string& equation) { + auto n_part = paddle::string::split_string(equation, "->").size(); + PADDLE_ENFORCE_EQ(n_part, + 2, + phi::errors::InvalidArgument( + "Required at least one `->` in equation of EinsumOp.")); + size_t pos; + auto trimed_equ = equation; + if ((pos = trimed_equ.find("->", 0)) != std::string::npos) { + trimed_equ.replace(pos, 2, "."); + } + auto is_valid_char = [](char c) { + if (c >= 'a' && c <= 'z') return true; + if (c == '.' || c == ',') return true; + return false; + }; + for (auto c : trimed_equ) { + if (!is_valid_char(c)) + PADDLE_THROW(phi::errors::InvalidArgument( + "Found invalid char in equation. Einsum only accept `a`-`z` and `...`" + "but get:`%c`", + c)); + } +} + +enum LabelType { + ALL_TYPE = 0, + Batch = 1, // ABO + Free, // AO, BO + Contraction, // AB + Reduction, // A, B +}; + +// map a label('a' - 'z') -> int, O(1) speed. +class LabelMap { + constexpr static int N = + 26 + 1; // 'a' - 'z' + '.', '.' is for broadcast dims + int default_value; + int map[N]; + + public: + explicit LabelMap(int default_value = 0) { + this->default_value = default_value; + for (int i = 0; i < N; ++i) map[i] = default_value; + } + int& operator[](int label) { + int i = label - 'a'; + if (label == '.') i = N - 1; + return map[i]; + } + int operator[](int label) const { + int i = label - 'a'; + if (label == '.') i = N - 1; + return map[i]; + } + // non-exist is present by is_default + bool is_default(char label) { + return (*this)[static_cast(label)] == default_value; + } +}; + +inline std::string label_to_string(const std::vector& all_labels, + const LabelMap& label2type) { + std::string str; + for (int a : all_labels) { + std::stringstream ss; + ss << label2type[a]; + str += ss.str(); + } + return str; +} + +inline static void ReplaceEllipsis(std::string& s) { // NOLINT + size_t pos; + if ((pos = s.find("...", 0)) != std::string::npos) { + s.replace(pos, 3, "."); + } + // remove all the space in the expression + while ((pos = s.find(" ", 0)) != std::string::npos) { + s.replace(pos, 1, ""); + } +} + +inline std::vector union_labels(const std::vector& a, + const std::vector& b) { + LabelMap counter(0); + std::vector res; + auto f = [&](char c) { + if (counter[static_cast(c)] == 0) { + res.push_back(c); + } + counter[static_cast(c)] += 1; + }; + std::for_each(a.begin(), a.end(), f); + std::for_each(b.begin(), b.end(), f); + return res; +} + +inline static void GlobalInfo(const std::vector& op_labels, + const std::string& right, + LabelMap* label2type, + std::vector* sorted_labels) { + // sorted_labels: ['.', , ] + VLOG(5) << "GlobalInfo: " + << paddle::string::join_strings(*sorted_labels, ","); + std::vector all; + LabelMap counter(0); + for (auto& ch : right) { // char + int c = ch; + (*label2type)[c] = LabelType::Free; + } + + for (auto& op : op_labels) { + for (auto& ch : op) { // char + int c = ch; + if (counter.is_default(c)) { + all.push_back(ch); + } + counter[c] += 1; + if ((*label2type)[c] != LabelType::Free && counter[c] == 2) + (*label2type)[c] = LabelType::Contraction; + else if (counter[c] == 2) + (*label2type)[c] = LabelType::Batch; + } + } + (*label2type)['.'] = LabelType::Batch; + std::for_each(all.begin(), all.end(), [sorted_labels, label2type](int c) { + if ((*label2type)[c] == LabelType::Batch) + sorted_labels->push_back(static_cast(c)); + }); + std::for_each(all.begin(), all.end(), [sorted_labels, label2type](int c) { + if ((*label2type)[c] == LabelType::Free) + sorted_labels->push_back(static_cast(c)); + }); + std::for_each(all.begin(), all.end(), [sorted_labels, label2type](int c) { + if ((*label2type)[c] == LabelType::Contraction) + sorted_labels->push_back(static_cast(c)); + }); + std::for_each(all.begin(), all.end(), [&sorted_labels, label2type](int c) { + if ((*label2type)[c] == LabelType::Reduction) + sorted_labels->push_back(static_cast(c)); + }); + VLOG(5) << "GlobalInfo: sorted_labels before: " + << paddle::string::join_strings(*sorted_labels, ","); + if (counter[static_cast('.')] > 0) { + std::vector tmp; + tmp.push_back('.'); + // push '.' in the front + *sorted_labels = union_labels(tmp, *sorted_labels); + VLOG(5) << "GlobalInfo: sorted_labels after: " + << paddle::string::join_strings(*sorted_labels, ","); + } +} + +inline static void InferLabelShape(const std::vector& op_labels, + const std::vector& inputs, + LabelMap* labelshape, + std::vector>* ellipsis_dims, + std::vector* broadcast_dims) { + VLOG(5) << "Start InferLabelShape"; + int n_broadcast_dims = 0; + for (size_t i = 0; i < op_labels.size(); ++i) { + VLOG(5) << "oplabels: " << op_labels[i]; + int valid_indices = std::count_if(op_labels[i].begin(), + op_labels[i].end(), + [](char c) { return c != '.'; }); + int n_ellipsis = inputs[i].size() - valid_indices; + VLOG(5) << "valid indices and n_ellipsis: " << valid_indices << " " + << n_ellipsis; + ellipsis_dims->at(i).resize(n_ellipsis); + n_broadcast_dims = std::max(n_broadcast_dims, n_ellipsis); + } + VLOG(5) << "InferLabelShape: Broadcast ndims:" << n_broadcast_dims; + *broadcast_dims = std::vector(n_broadcast_dims, 1); + + for (size_t i = 0; i < op_labels.size(); ++i) { + auto& op_str = op_labels[i]; + auto& op_dim = inputs[i]; + int dim_ptr = 0; + for (int c : op_str) { + if (c == '.') { + for (auto& v : ellipsis_dims->at(i)) { + v = op_dim[dim_ptr]; + dim_ptr++; + } + } else if (labelshape->is_default(c) || (*labelshape)[c] == -1) { + (*labelshape)[c] = op_dim[dim_ptr]; + dim_ptr++; + } else { + PADDLE_ENFORCE_EQ( + (*labelshape)[c], + op_dim[dim_ptr], + phi::errors::InvalidArgument( + "Same label have different shapes for label: `%c`", c)); + dim_ptr++; + } + } + } + for (size_t i = 0; i < op_labels.size(); ++i) { + VLOG(5) << "InferLabelShape: Ellipsis ndims:" + << paddle::string::join_strings(ellipsis_dims->at(i), ","); + int idx = n_broadcast_dims - ellipsis_dims->at(i).size(); + for (auto v : ellipsis_dims->at(i)) { + PADDLE_ENFORCE_EQ( + v == 1 || broadcast_dims->at(idx) == 1 || + broadcast_dims->at(idx) == v, + true, + phi::errors::InvalidArgument( + "Ellipsis dims can't broadcasts. Please Check you operands.")); + broadcast_dims->at(idx) = std::max(v, broadcast_dims->at(idx)); + idx += 1; + } + } + VLOG(5) << "InferLabelShape: Broadcast dims:" + << paddle::string::join_strings(*broadcast_dims, ","); +} + +inline static void InferLabelPerm(const std::string& op, + int n_broadcast, + LabelMap* label2perm) { + int cur = 0; + for (int c : op) { + (*label2perm)[c] = cur; + if (c == '.') { + cur += n_broadcast; + } else { + cur += 1; + } + } +} + +inline static void InferOutputDims(const std::string& right, + const std::vector& broadcast_dims, + const LabelMap& labelshape, + std::vector* output_dims) { + for (int c : right) { + if (c == '.') { + output_dims->insert( + output_dims->end(), broadcast_dims.begin(), broadcast_dims.end()); + } else { + output_dims->push_back(labelshape[c]); + } + } +} +// +inline static void ParseEinsumEquation( + const std::string& equation, + const std::vector& inputs, + LabelMap* labelshape, + LabelMap* labeltype, + std::vector* all_labels, + std::vector* label2perms, + std::vector>* ellipsis_dims, + std::vector* broadcast_dims, + std::vector* output_dims, + std::string* right) { + auto results = paddle::string::split_string(equation, "->"); + auto left = results[0]; + ReplaceEllipsis(left); + *right = results[1].substr(1); + ReplaceEllipsis(*right); + auto op_labels = paddle::string::split_string(left, ","); + std::for_each(op_labels.begin(), op_labels.end(), ReplaceEllipsis); + GlobalInfo(op_labels, *right, labeltype, all_labels); + InferLabelShape(op_labels, inputs, labelshape, ellipsis_dims, broadcast_dims); + VLOG(5) << "Einsum Infershape: right:" << right; + VLOG(5) << "Einsum Infershape: op_labels:" + << paddle::string::join_strings(op_labels, "\n"); + InferOutputDims(*right, *broadcast_dims, *labelshape, output_dims); + for (size_t i = 0; i < inputs.size(); ++i) { + InferLabelPerm( + op_labels[i], ellipsis_dims->at(i).size(), &((*label2perms)[i])); + } +} + +inline void EinsumInferShape(const std::vector& inputs, + const std::string& equation, + MetaTensor* out) { + // collect the following informations to prepare einsum. + LabelMap labelshape(0); + LabelMap labeltype(LabelType::Reduction); + std::vector label2perms(inputs.size(), LabelMap(-1)); + std::vector all_labels; + std::vector broadcast_dims; + std::vector output_dims; + std::vector> ellipsis_dims(2); + + std::vector input_dims; + for (auto& i : inputs) { + input_dims.push_back(i->dims()); + } + std::string right; + ParseEinsumEquation(equation, + input_dims, + &labelshape, + &labeltype, + &all_labels, + &label2perms, + &ellipsis_dims, + &broadcast_dims, + &output_dims, + &right); + + VLOG(3) << "Einsum Infershape: input dims:" + << paddle::string::join_strings(input_dims, "\n"); + VLOG(3) << "Einsum Infershape: equation:" << equation; + VLOG(3) << "Einsum Infershape: all_labels:" + << paddle::string::join_strings(all_labels, ","); + VLOG(3) << "Einsum Infershape: output dims:" + << paddle::string::join_strings(output_dims, ","); + VLOG(3) << "Label Type is : " << label_to_string(all_labels, labeltype); + VLOG(3) << "Label Shape is : " << label_to_string(all_labels, labelshape); +} + +template +std::vector GetLabelIndexByType(const std::vector& all_labels, + const LabelMap& type, + const LabelMap& perm, + const std::vector& ellipsis, + LabelType filter) { + std::vector res; + for (T c : all_labels) { + if ((filter == LabelType::ALL_TYPE || type[c] == filter) && perm[c] != -1) { + if (c == '.') { + for (size_t i = 0; i < ellipsis.size(); ++i) res.push_back(perm[c] + i); + } else { + res.push_back(perm[c]); + } + } + } + return res; +} + +template +std::vector GetShapeByType(const std::vector& all_labels, + const LabelMap& type, + const LabelMap& perm, + const LabelMap& label2shape, + const std::vector& ellipsis, + LabelType filter) { + std::vector res; + for (T c : all_labels) { + if ((filter == LabelType::ALL_TYPE || type[c] == filter) && perm[c] != -1) { + if (c == '.') + res.insert(res.end(), ellipsis.begin(), ellipsis.end()); + else + res.push_back(label2shape[c]); + } + } + return res; +} + +template +DenseTensor PerformReduction(const Context& dev_ctx, + const DenseTensor& tensor, + const LabelMap& label2perm, + const std::vector& all_labels, + const std::vector& ellipsis, + const LabelMap& label2type) { + auto indices = GetLabelIndexByType( + all_labels, label2type, label2perm, ellipsis, LabelType::Reduction); + VLOG(5) << "call PerformReduction: with axis: " + << paddle::string::join_strings(indices, ","); + if (indices.size() == 0) return tensor; + return Sum(dev_ctx, tensor, indices, tensor.dtype(), true); +} + +template +DenseTensor PerformTranspose(const Context& dev_ctx, + const DenseTensor& tensor, + const LabelMap& label2perm, + const std::vector& all_labels, + const std::vector& ellipsis, + const LabelMap& label2type) { + auto is_no_need_transpose = [](std::vector& axis) { + for (size_t i = 0; i < axis.size(); ++i) { + if (i != size_t(axis[i])) return false; + } + return true; + }; + auto axis = GetLabelIndexByType( + all_labels, label2type, label2perm, ellipsis, LabelType::ALL_TYPE); + VLOG(5) << "PerformTranspose: " << paddle::string::join_strings(axis, ","); + if (is_no_need_transpose(axis)) { + return tensor; + } + auto ret = Transpose(dev_ctx, tensor, axis); + VLOG(5) << "PerformTranspose: do_transpose()"; + return ret; +} + +template +DenseTensor PerformContraction( + const Context& dev_ctx, + const DenseTensor& A, + const DenseTensor& B, + const std::vector& label2perm, + const std::vector& all_labels, + const LabelMap& label2type, + const LabelMap& label2shape, + const std::vector>& ellipsis_dims, + const std::vector& broadcast_dims) { + // Get All the Batches, so perm is + auto all_valid = LabelMap(1); + auto recover_dim = GetShapeByType(all_labels, + label2type, + all_valid, + label2shape, + broadcast_dims, + LabelType::Batch); + auto preprocess = [&](const DenseTensor& t, + const LabelMap& perm, + const std::vector& ellipsis) -> DenseTensor { + auto frees = GetShapeByType( + all_labels, label2type, perm, label2shape, ellipsis, LabelType::Free); + auto conts = GetShapeByType(all_labels, + label2type, + perm, + label2shape, + ellipsis, + LabelType::Contraction); + auto trans_t = PerformTranspose( + dev_ctx, t, perm, all_labels, ellipsis, label2type); + auto mul_dims = GetShapeByType( + all_labels, label2type, perm, label2shape, ellipsis, LabelType::Batch); + recover_dim.insert(recover_dim.end(), frees.begin(), frees.end()); + mul_dims.push_back( + std::accumulate(frees.begin(), frees.end(), 1, std::multiplies())); + mul_dims.push_back( + std::accumulate(conts.begin(), conts.end(), 1, std::multiplies())); + VLOG(5) << "PerformContraction: mul_dims: " + << paddle::string::join_strings(mul_dims, ","); + trans_t.Resize(make_ddim(mul_dims)); + return trans_t; + }; + auto trans_a = preprocess(A, label2perm[0], ellipsis_dims[0]); + auto trans_b = preprocess(B, label2perm[1], ellipsis_dims[1]); + auto after_contraction = + Matmul(dev_ctx, trans_a, trans_b, false, true); + VLOG(5) << "PerformContraction: recover_dim: " + << paddle::string::join_strings(recover_dim, ","); + after_contraction.Resize(make_ddim(recover_dim)); + return after_contraction; +} + +template +void TransposeToOutput(const Context& dev_ctx, + const DenseTensor& to_trans, + const std::string& right, + const std::vector& all_labels, + int n_broadcast_dims, + DenseTensor* output) { + std::vector axis; + int offset = 0; + if (std::find(all_labels.begin(), all_labels.end(), '.') != + all_labels.end()) { + offset = n_broadcast_dims - 1; + } + for (char c : right) { + if (c == '.') { + for (int i = 0; i < n_broadcast_dims; ++i) axis.push_back(i); + } else { + auto it = std::find(all_labels.begin(), all_labels.end(), c); + PADDLE_ENFORCE_NE(it, + all_labels.end(), + phi::errors::InvalidArgument("Must in all_labels.")); + axis.push_back(it - all_labels.begin() + offset); + } + } + VLOG(5) << "call TransposeToOutput: with axis: " + << paddle::string::join_strings(axis, ","); + if (axis.size() == 0) return output->ShareBufferWith(to_trans); + return TransposeKernel(dev_ctx, to_trans, axis, output); +} + +template +void EinsumKernel(const Context& dev_ctx, + const std::vector& inputs, + const std::string& equation, + DenseTensor* out) { + ValidationCheck(equation); + // collect the following informations to prepare einsum. + LabelMap labelshape(0); + LabelMap labeltype(LabelType::Reduction); + std::vector label2perms(inputs.size(), LabelMap(-1)); + std::vector all_labels; // order: ABO, AO, BO, AB, Reduce + std::vector> ellipsis_dims(2); + std::vector broadcast_dims; + std::vector output_dims; + + std::vector input_dims; + for (auto& i : inputs) { + input_dims.push_back(i->dims()); + } + std::string right; + ParseEinsumEquation(equation, + input_dims, + &labelshape, + &labeltype, + &all_labels, + &label2perms, + &ellipsis_dims, + &broadcast_dims, + &output_dims, + &right); + out->Resize(make_ddim(output_dims)); + if (inputs.size() == 2) { + auto& A = inputs[0]; + auto& B = inputs[1]; + // Reduce Procedure + auto reduce_A = PerformReduction( + dev_ctx, *A, label2perms[0], all_labels, ellipsis_dims[0], labeltype); + auto reduce_B = PerformReduction( + dev_ctx, *B, label2perms[1], all_labels, ellipsis_dims[1], labeltype); + // Contract Procedure + dev_ctx.template Alloc(out); + auto after_contraction = PerformContraction(dev_ctx, + reduce_A, + reduce_B, + label2perms, + all_labels, + labeltype, + labelshape, + ellipsis_dims, + broadcast_dims); + TransposeToOutput(dev_ctx, + after_contraction, + right, + all_labels, + broadcast_dims.size(), + out); + // Reshape Procedure + } else if (inputs.size() == 1) { + auto reduce_A = PerformReduction(dev_ctx, + *inputs[0], + label2perms[0], + all_labels, + ellipsis_dims[0], + labeltype); + std::vector right_labels; + for (auto c : right) right_labels.push_back(c); + right_labels = union_labels(right_labels, all_labels); + *out = PerformTranspose(dev_ctx, + reduce_A, + label2perms[0], + right_labels, + broadcast_dims, + labeltype); + out->Resize(make_ddim(output_dims)); + } else { + PADDLE_THROW(phi::errors::InvalidArgument( + "EinsumOp kernel only support len(operands) between (0, 2]. Use " + "opt_einsum first to convert multi-variable to binary-variable.")); + } +} + +} // namespace phi diff --git a/paddle/phi/kernels/impl/elementwise_grad_kernel_impl.h b/paddle/phi/kernels/impl/elementwise_grad_kernel_impl.h index aba4a5f5fbd43..fa1f15672b903 100644 --- a/paddle/phi/kernels/impl/elementwise_grad_kernel_impl.h +++ b/paddle/phi/kernels/impl/elementwise_grad_kernel_impl.h @@ -360,6 +360,14 @@ struct MulGradDX { HOSTDEVICE T operator()(T x, T y, T out, T dout) const { return dout * y; } }; +// avoid [-Wint-in-bool-context] warning +template <> +struct MulGradDX { + HOSTDEVICE bool operator()(bool x, bool y, bool out, bool dout) const { + return dout && y; + } +}; + template struct MulGradDX> { HOSTDEVICE phi::dtype::complex operator()( @@ -383,6 +391,14 @@ struct MulGradDY { HOSTDEVICE T operator()(T x, T y, T out, T dout) const { return dout * x; } }; +// avoid [-Wint-in-bool-context] warning +template <> +struct MulGradDY { + HOSTDEVICE bool operator()(bool x, bool y, bool out, bool dout) const { + return dout && x; + } +}; + template struct MulGradDY> { HOSTDEVICE phi::dtype::complex operator()( diff --git a/paddle/phi/kernels/impl/elementwise_kernel_impl.h b/paddle/phi/kernels/impl/elementwise_kernel_impl.h index b126ca9b84227..4f1e7af582c96 100644 --- a/paddle/phi/kernels/impl/elementwise_kernel_impl.h +++ b/paddle/phi/kernels/impl/elementwise_kernel_impl.h @@ -55,7 +55,9 @@ namespace phi { int axis, \ DenseTensor* out) { \ std::vector inputs; \ + inputs.reserve(2); \ std::vector outputs; \ + outputs.reserve(1); \ inputs.emplace_back(&x); \ inputs.emplace_back(&y); \ outputs.emplace_back(out); \ diff --git a/paddle/phi/kernels/impl/pixel_unshuffle_grad_kernel_impl.h b/paddle/phi/kernels/impl/pixel_unshuffle_grad_kernel_impl.h new file mode 100644 index 0000000000000..cb02539f2e890 --- /dev/null +++ b/paddle/phi/kernels/impl/pixel_unshuffle_grad_kernel_impl.h @@ -0,0 +1,58 @@ +// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#include +#include + +#include "paddle/phi/core/dense_tensor.h" +#include "paddle/phi/kernels/funcs/math_function.h" + +namespace phi { + +template +void PixelUnshuffleGradKernel(const Context& dev_ctx, + const DenseTensor& out_grad, + int downscale_factor, + const std::string& data_format, + DenseTensor* x_grad) { + auto* dout = &out_grad; + auto* dx = x_grad; + dev_ctx.template Alloc(dx); + int factor = downscale_factor; + bool channel_last = (data_format == "NHWC"); + auto do_dims = dout->dims(); + auto dx_dims = dx->dims(); + + DenseTensor t(*dout); + if (!channel_last) { + t.Resize({do_dims[0], dx_dims[1], factor, factor, do_dims[2], do_dims[3]}); + } else { + t.Resize({do_dims[0], do_dims[1], do_dims[2], dx_dims[3], factor, factor}); + } + std::vector axis = {0, 1, 4, 2, 5, 3}; + + DenseTensor o(*dx); + if (!channel_last) { + o.Resize({do_dims[0], dx_dims[1], do_dims[2], factor, do_dims[3], factor}); + } else { + o.Resize({do_dims[0], do_dims[1], factor, do_dims[2], factor, dx_dims[3]}); + } + phi::funcs::Transpose trans; + trans(dev_ctx, t, &o, axis); + dx->Resize(dx_dims); +} + +} // namespace phi diff --git a/paddle/phi/kernels/impl/pixel_unshuffle_kernel_impl.h b/paddle/phi/kernels/impl/pixel_unshuffle_kernel_impl.h new file mode 100644 index 0000000000000..0a140b270ba1b --- /dev/null +++ b/paddle/phi/kernels/impl/pixel_unshuffle_kernel_impl.h @@ -0,0 +1,57 @@ +// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#include +#include + +#include "paddle/phi/core/dense_tensor.h" +#include "paddle/phi/kernels/funcs/math_function.h" + +namespace phi { + +template +void PixelUnshuffleKernel(const Context& dev_ctx, + const DenseTensor& x, + int downscale_factor, + const std::string& data_format, + DenseTensor* out) { + auto* in = &x; + dev_ctx.template Alloc(out); + int factor = downscale_factor; + bool channel_last = (data_format == "NHWC"); + auto in_dims = in->dims(); + auto o_dims = out->dims(); + + DenseTensor t(*in); + if (!channel_last) { + t.Resize({in_dims[0], in_dims[1], o_dims[2], factor, o_dims[3], factor}); + } else { + t.Resize({in_dims[0], o_dims[1], factor, o_dims[2], factor, in_dims[3]}); + } + std::vector axis = {0, 1, 3, 5, 2, 4}; + + DenseTensor o(*out); + if (!channel_last) { + o.Resize({in_dims[0], in_dims[1], factor, factor, o_dims[2], o_dims[3]}); + } else { + o.Resize({in_dims[0], o_dims[1], o_dims[2], in_dims[3], factor, factor}); + } + phi::funcs::Transpose trans; + trans(dev_ctx, t, &o, axis); + out->Resize(o_dims); +} + +} // namespace phi diff --git a/paddle/phi/kernels/kps/reduce_sum_kernel.cu b/paddle/phi/kernels/kps/reduce_sum_kernel.cu index 6c039897ddd30..e800e4685ec04 100644 --- a/paddle/phi/kernels/kps/reduce_sum_kernel.cu +++ b/paddle/phi/kernels/kps/reduce_sum_kernel.cu @@ -27,6 +27,9 @@ void SumRawKernel(const Context& dev_ctx, bool reduce_all, DataType out_dtype, DenseTensor* out) { + if (out_dtype == DataType::UNDEFINED && out->dtype() != x.dtype()) { + out_dtype = out->dtype(); + } phi::Reduce( dev_ctx, x, reduce_all, dims, keep_dim, out_dtype, out); } diff --git a/paddle/phi/kernels/pixel_unshuffle_grad_kernel.h b/paddle/phi/kernels/pixel_unshuffle_grad_kernel.h new file mode 100644 index 0000000000000..868633e56be50 --- /dev/null +++ b/paddle/phi/kernels/pixel_unshuffle_grad_kernel.h @@ -0,0 +1,29 @@ +// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#include +#include "paddle/phi/core/dense_tensor.h" + +namespace phi { + +template +void PixelUnshuffleGradKernel(const Context& dev_ctx, + const DenseTensor& out_grad, + int downscale_factor, + const std::string& data_format, + DenseTensor* x_grad); + +} // namespace phi diff --git a/paddle/phi/kernels/pixel_unshuffle_kernel.h b/paddle/phi/kernels/pixel_unshuffle_kernel.h new file mode 100644 index 0000000000000..179e2b6639f9e --- /dev/null +++ b/paddle/phi/kernels/pixel_unshuffle_kernel.h @@ -0,0 +1,29 @@ +// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#include +#include "paddle/phi/core/dense_tensor.h" + +namespace phi { + +template +void PixelUnshuffleKernel(const Context& dev_ctx, + const DenseTensor& x, + int downscale_factor, + const std::string& data_format, + DenseTensor* out); + +} // namespace phi diff --git a/paddle/phi/kernels/shape_kernel.cc b/paddle/phi/kernels/shape_kernel.cc index dd26a7edc9cdd..f87b5014c1207 100644 --- a/paddle/phi/kernels/shape_kernel.cc +++ b/paddle/phi/kernels/shape_kernel.cc @@ -63,5 +63,7 @@ PD_REGISTER_KERNEL(shape, double, phi::dtype::complex, phi::dtype::complex, - phi::dtype::float16) {} + phi::dtype::float16) { + kernel->InputAt(0).SetBackend(phi::Backend::ALL_BACKEND); +} #endif diff --git a/paddle/phi/kernels/sparse/cpu/coalesced_kernel.cc b/paddle/phi/kernels/sparse/cpu/coalesced_kernel.cc index 0ebddf9b683f0..22c5e14b35f56 100644 --- a/paddle/phi/kernels/sparse/cpu/coalesced_kernel.cc +++ b/paddle/phi/kernels/sparse/cpu/coalesced_kernel.cc @@ -44,7 +44,7 @@ void CoalescedCPUKernel(const CPUContext& dev_ctx, const T* x_values_ptr = x_values.data(); const int64_t stride = - x.dims().size() == sparse_dim ? 1 : x.dims().size() - sparse_dim; + x.dims().size() == sparse_dim ? 1 : x.non_zero_elements().dims()[1]; std::map> indices_to_index; for (uint64_t i = 0; i < x_indexs.size(); i++) { diff --git a/paddle/phi/kernels/sparse/cpu/sparse_mask_kernel.cc b/paddle/phi/kernels/sparse/cpu/sparse_mask_kernel.cc index 1508de407caa7..0ec8b808ba838 100644 --- a/paddle/phi/kernels/sparse/cpu/sparse_mask_kernel.cc +++ b/paddle/phi/kernels/sparse/cpu/sparse_mask_kernel.cc @@ -125,7 +125,7 @@ void SparseMaskHelperCPUKernel(const CPUContext& dev_ctx, T* out_ptr = out->data(); memset(out_ptr, static_cast(0), out->numel() * sizeof(T)); const int64_t stride = - x.dims().size() == sparse_dim ? 1 : x.dims().size() - sparse_dim; + x.dims().size() == sparse_dim ? 1 : x.non_zero_elements().dims()[1]; const T* in_ptr = x.non_zero_elements().data(); // TODO(zhangkaihuo): multithreading can be used for acceleration for (uint64_t i = 0; i < mask_indexs.size(); i++) { diff --git a/paddle/phi/kernels/sparse/cpu/sparse_utils_kernel.cc b/paddle/phi/kernels/sparse/cpu/sparse_utils_kernel.cc index 0499371a4dd17..685aa6b30bdc1 100644 --- a/paddle/phi/kernels/sparse/cpu/sparse_utils_kernel.cc +++ b/paddle/phi/kernels/sparse/cpu/sparse_utils_kernel.cc @@ -256,9 +256,11 @@ void SparseCooToDenseKernel(const Context& dev_ctx, } const int64_t dense_dim = values.dims().size() - 1; - const auto place = dev_ctx.GetPlace(); const T* x_data = values.data(); - T* out_data = out->mutable_data(place); + *out = phi::Empty( + dev_ctx, + DenseTensorMeta(x.dtype(), x.dims(), x.non_zero_elements().layout())); + T* out_data = out->data(); int64_t base_offset = 1; for (int64_t i = 0; i < dense_dim; i++) { base_offset *= dense_dims[sparse_dim + i]; diff --git a/paddle/phi/kernels/sparse/gpu/coalesced_kernel.cu b/paddle/phi/kernels/sparse/gpu/coalesced_kernel.cu index 3ffcd28955a53..b2e7884580c74 100644 --- a/paddle/phi/kernels/sparse/gpu/coalesced_kernel.cu +++ b/paddle/phi/kernels/sparse/gpu/coalesced_kernel.cu @@ -76,7 +76,7 @@ void CoalescedGPUKernel(const GPUContext& dev_ctx, // 2. get the address of each non-zero values const T* x_values_ptr = x_values.data(); const int64_t stride = - x.dims().size() == sparse_dim ? 1 : x.dims().size() - sparse_dim; + x.dims().size() == sparse_dim ? 1 : x.non_zero_elements().dims()[1]; DenseTensor values_indexs = phi::Empty( dev_ctx, DenseTensorMeta(DataType::INT32, {nnz}, DataLayout::NCHW)); int* values_indexs_ptr = values_indexs.data(); diff --git a/paddle/phi/kernels/sparse/gpu/sparse_mask_kernel.cu b/paddle/phi/kernels/sparse/gpu/sparse_mask_kernel.cu index 4e2d12f33955e..4253845956ea7 100644 --- a/paddle/phi/kernels/sparse/gpu/sparse_mask_kernel.cu +++ b/paddle/phi/kernels/sparse/gpu/sparse_mask_kernel.cu @@ -231,7 +231,7 @@ void SparseMaskHelperGPUKernel(const GPUContext& dev_ctx, T* out_ptr = out->data(); const int64_t stride = - x.dims().size() == sparse_dim ? 1 : x.dims().size() - sparse_dim; + x.dims().size() == sparse_dim ? 1 : x.non_zero_elements().dims()[1]; SparseMaskCopyKernel<<non_zero_elements().numel(), - static_cast(-FLT_MAX)); + static_cast(0)); // TODO(zhangkaihuo) Replacing multiple calls with one kernel may be faster for (int i = 0; i < kernel_size; i++) { if (counter[i] <= 0) { diff --git a/paddle/phi/kernels/sparse/gpu/sparse_utils_kernel.cu b/paddle/phi/kernels/sparse/gpu/sparse_utils_kernel.cu index 0b6ac1aed0147..960d7eab26463 100644 --- a/paddle/phi/kernels/sparse/gpu/sparse_utils_kernel.cu +++ b/paddle/phi/kernels/sparse/gpu/sparse_utils_kernel.cu @@ -503,7 +503,10 @@ void SparseCooToDenseKernel(const Context& dev_ctx, const auto place = dev_ctx.GetPlace(); const T* x_data = values.data(); - T* out_data = out->mutable_data(place); + *out = phi::Empty(dev_ctx, + phi::DenseTensorMeta( + x.dtype(), x.dims(), x.non_zero_elements().layout())); + T* out_data = out->data(); int64_t base_offset = 1; for (int64_t i = 0; i < dense_dim; i++) { base_offset *= dense_dims[sparse_dim + i]; diff --git a/paddle/phi/kernels/sparse/sparse_utils_kernel.h b/paddle/phi/kernels/sparse/sparse_utils_kernel.h index 072e6f141f8f1..d39790fcea5e3 100644 --- a/paddle/phi/kernels/sparse/sparse_utils_kernel.h +++ b/paddle/phi/kernels/sparse/sparse_utils_kernel.h @@ -110,7 +110,7 @@ void SparseCooToDenseKernel(const Context& dev_ctx, template DenseTensor SparseCooToDense(const Context& dev_ctx, const SparseCooTensor& x) { - DenseTensorMeta meta(x.dtype(), x.dims(), x.layout()); + DenseTensorMeta meta(x.dtype(), x.dims(), x.non_zero_elements().layout()); DenseTensor dense = phi::Empty(dev_ctx, std::move(meta)); SparseCooToDenseKernel(dev_ctx, x, &dense); return dense; @@ -129,7 +129,7 @@ void SparseCsrToDenseKernel(const Context& dev_ctx, template DenseTensor SparseCsrToDense(const Context& dev_ctx, const SparseCsrTensor& x) { - DenseTensorMeta meta(x.dtype(), x.dims(), x.layout()); + DenseTensorMeta meta(x.dtype(), x.dims(), x.non_zero_elements().layout()); DenseTensor dense = phi::Empty(dev_ctx, std::move(meta)); SparseCsrToDenseKernel(dev_ctx, x, &dense); return dense; diff --git a/paddle/phi/kernels/where_grad_kernel.h b/paddle/phi/kernels/where_grad_kernel.h index 1a3c66ee6ed84..5f596da93e9c2 100644 --- a/paddle/phi/kernels/where_grad_kernel.h +++ b/paddle/phi/kernels/where_grad_kernel.h @@ -14,10 +14,7 @@ #pragma once -#include "paddle/phi/backends/all_context.h" -#include "paddle/phi/backends/gpu/gpu_launch_config.h" #include "paddle/phi/core/dense_tensor.h" -#include "paddle/phi/core/kernel_registry.h" namespace phi { diff --git a/paddle/phi/kernels/where_kernel.h b/paddle/phi/kernels/where_kernel.h index 254271ac9c723..6348177e69764 100644 --- a/paddle/phi/kernels/where_kernel.h +++ b/paddle/phi/kernels/where_kernel.h @@ -14,10 +14,7 @@ #pragma once -#include "paddle/phi/backends/all_context.h" -#include "paddle/phi/backends/gpu/gpu_launch_config.h" #include "paddle/phi/core/dense_tensor.h" -#include "paddle/phi/core/kernel_registry.h" namespace phi { diff --git a/paddle/phi/ops/compat/adam_sig.cc b/paddle/phi/ops/compat/adam_sig.cc index 958538cd7dfc2..f3e7eeb6b6762 100644 --- a/paddle/phi/ops/compat/adam_sig.cc +++ b/paddle/phi/ops/compat/adam_sig.cc @@ -19,22 +19,22 @@ namespace phi { KernelSignature AdamOpArgumentMapping(const ArgumentMappingContext& ctx) { - paddle::SmallVector in_names = {"Param", - "Grad", - "LearningRate", - "Moment1", - "Moment2", - "Beta1Pow", - "Beta2Pow", - "MasterParam", - "SkipUpdate"}; - paddle::SmallVector out_names = {"ParamOut", - "Moment1Out", - "Moment2Out", - "Beta1PowOut", - "Beta2PowOut", - "MasterParamOut"}; - paddle::SmallVector attr_names; + paddle::small_vector in_names = {"Param", + "Grad", + "LearningRate", + "Moment1", + "Moment2", + "Beta1Pow", + "Beta2Pow", + "MasterParam", + "SkipUpdate"}; + paddle::small_vector out_names = {"ParamOut", + "Moment1Out", + "Moment2Out", + "Beta1PowOut", + "Beta2PowOut", + "MasterParamOut"}; + paddle::small_vector attr_names; attr_names.emplace_back(ctx.HasInput("Beta1Tensor") ? "Beta1Tensor" : "beta1"); diff --git a/paddle/phi/ops/compat/adamw_sig.cc b/paddle/phi/ops/compat/adamw_sig.cc index e417aa30ba493..b4cf6f3cbbe6d 100644 --- a/paddle/phi/ops/compat/adamw_sig.cc +++ b/paddle/phi/ops/compat/adamw_sig.cc @@ -19,22 +19,22 @@ namespace phi { KernelSignature AdamwOpArgumentMapping(const ArgumentMappingContext& ctx) { - paddle::SmallVector in_names = {"Param", - "Grad", - "LearningRate", - "Moment1", - "Moment2", - "Beta1Pow", - "Beta2Pow", - "MasterParam", - "SkipUpdate"}; - paddle::SmallVector out_names = {"ParamOut", - "Moment1Out", - "Moment2Out", - "Beta1PowOut", - "Beta2PowOut", - "MasterParamOut"}; - paddle::SmallVector attr_names; + paddle::small_vector in_names = {"Param", + "Grad", + "LearningRate", + "Moment1", + "Moment2", + "Beta1Pow", + "Beta2Pow", + "MasterParam", + "SkipUpdate"}; + paddle::small_vector out_names = {"ParamOut", + "Moment1Out", + "Moment2Out", + "Beta1PowOut", + "Beta2PowOut", + "MasterParamOut"}; + paddle::small_vector attr_names; attr_names.emplace_back(ctx.HasInput("Beta1Tensor") ? "Beta1Tensor" : "beta1"); diff --git a/paddle/phi/ops/compat/channel_shuffle_sig.cc b/paddle/phi/ops/compat/channel_shuffle_sig.cc new file mode 100644 index 0000000000000..ae0aa0a80b6f0 --- /dev/null +++ b/paddle/phi/ops/compat/channel_shuffle_sig.cc @@ -0,0 +1,30 @@ +// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "paddle/phi/core/compat/op_utils.h" + +namespace phi { + +KernelSignature ChannelShuffleGradOpArgumentMapping( + const ArgumentMappingContext& ctx) { + return KernelSignature("channel_shuffle_grad", + {"Out@GRAD"}, + {"groups", "data_format"}, + {"X@GRAD"}); +} + +} // namespace phi + +PD_REGISTER_ARG_MAPPING_FN(channel_shuffle_grad, + phi::ChannelShuffleGradOpArgumentMapping); diff --git a/paddle/phi/ops/compat/clip_sig.cc b/paddle/phi/ops/compat/clip_sig.cc index 25a34f2b9c89f..889dbf6ef9f79 100644 --- a/paddle/phi/ops/compat/clip_sig.cc +++ b/paddle/phi/ops/compat/clip_sig.cc @@ -18,7 +18,7 @@ namespace phi { KernelSignature ClipOpArgumentMapping(const ArgumentMappingContext& ctx) { - paddle::SmallVector attr_names; + paddle::small_vector attr_names; attr_names.emplace_back(ctx.HasInput("Min") ? "Min" : "min"); attr_names.emplace_back(ctx.HasInput("Max") ? "Max" : "max"); if (ctx.IsDenseTensorInput("X")) { diff --git a/paddle/phi/ops/compat/einsum_sig.cc b/paddle/phi/ops/compat/einsum_sig.cc new file mode 100644 index 0000000000000..0b3cc3425df45 --- /dev/null +++ b/paddle/phi/ops/compat/einsum_sig.cc @@ -0,0 +1,32 @@ +/* Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + +#include "paddle/phi/core/compat/op_utils.h" + +namespace phi { + +KernelSignature EinsumOpArgumentMapping(const ArgumentMappingContext& ctx) { + return KernelSignature("einsum", {"Operands"}, {"equation"}, {"Out"}); +} + +KernelSignature EinsumGradOpArgumentMapping(const ArgumentMappingContext& ctx) { + return KernelSignature("einsum_grad", + {"Operands", {"Out@GRAD"}}, + {"equation"}, + {{"Operands@GRAD"}}); +} +} // namespace phi + +PD_REGISTER_ARG_MAPPING_FN(einsum, phi::EinsumOpArgumentMapping); +PD_REGISTER_ARG_MAPPING_FN(einsum_grad, phi::EinsumGradOpArgumentMapping); diff --git a/paddle/phi/ops/compat/pixel_unshuffle_sig.cc b/paddle/phi/ops/compat/pixel_unshuffle_sig.cc new file mode 100644 index 0000000000000..817dc1a228877 --- /dev/null +++ b/paddle/phi/ops/compat/pixel_unshuffle_sig.cc @@ -0,0 +1,30 @@ +// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "paddle/phi/core/compat/op_utils.h" + +namespace phi { + +KernelSignature PixelUnshuffleGradOpArgumentMapping( + const ArgumentMappingContext& ctx) { + return KernelSignature("pixel_unshuffle_grad", + {"Out@GRAD"}, + {"downscale_factor", "data_format"}, + {"X@GRAD"}); +} + +} // namespace phi + +PD_REGISTER_ARG_MAPPING_FN(pixel_unshuffle_grad, + phi::PixelUnshuffleGradOpArgumentMapping); diff --git a/paddle/phi/ops/compat/strided_slice_sig.cc b/paddle/phi/ops/compat/strided_slice_sig.cc index 5421fcd616ce7..02b3914787866 100644 --- a/paddle/phi/ops/compat/strided_slice_sig.cc +++ b/paddle/phi/ops/compat/strided_slice_sig.cc @@ -48,14 +48,14 @@ KernelSignature StridedSliceOpArgumentMapping( ? (use_attr_strides ? "strides" : "StridesTensorList") : "strides"); - paddle::SmallVector inputs = {"Input"}; - paddle::SmallVector attrs = {"axes", - starts_key, - ends_key, - strides_key, - "infer_flags", - "decrease_axis"}; - paddle::SmallVector outputs = {"Out"}; + paddle::small_vector inputs = {"Input"}; + paddle::small_vector attrs = {"axes", + starts_key, + ends_key, + strides_key, + "infer_flags", + "decrease_axis"}; + paddle::small_vector outputs = {"Out"}; const char* kernel_name; if (ctx.IsDenseTensorVectorInput("Input")) { @@ -97,14 +97,14 @@ KernelSignature StridedSliceGradOpArgumentMapping( ? (use_attr_strides ? "strides" : "StridesTensorList") : "strides"); - paddle::SmallVector inputs = {"Input", "Out@GRAD"}; - paddle::SmallVector attrs = {"axes", - starts_key, - ends_key, - strides_key, - "infer_flags", - "decrease_axis"}; - paddle::SmallVector outputs = {"Input@GRAD"}; + paddle::small_vector inputs = {"Input", "Out@GRAD"}; + paddle::small_vector attrs = {"axes", + starts_key, + ends_key, + strides_key, + "infer_flags", + "decrease_axis"}; + paddle::small_vector outputs = {"Input@GRAD"}; const char* kernel_name; if (ctx.IsDenseTensorVectorInput("Input")) { diff --git a/paddle/phi/ops/compat/sum_sig.cc b/paddle/phi/ops/compat/sum_sig.cc index 4364047b0e61b..d71111408f854 100644 --- a/paddle/phi/ops/compat/sum_sig.cc +++ b/paddle/phi/ops/compat/sum_sig.cc @@ -18,7 +18,7 @@ namespace phi { KernelSignature SumOpArgumentMapping(const ArgumentMappingContext& ctx) { - if (ctx.IsDenseTensorInput("X")) { + if (ctx.IsDenseTensorInputs("X")) { return KernelSignature("add_n", {"X"}, {}, {"Out"}); } return KernelSignature("unregistered", {}, {}, {}); diff --git a/paddle/phi/tests/core/test_custom_kernel.cc b/paddle/phi/tests/core/test_custom_kernel.cc index 07530f70b7ab5..2a5b8ec8fa000 100644 --- a/paddle/phi/tests/core/test_custom_kernel.cc +++ b/paddle/phi/tests/core/test_custom_kernel.cc @@ -49,7 +49,6 @@ void FakeDot(const Context& dev_ctx, float fake_attr_float, double fake_attr_double, int64_t fake_attr_int64, - phi::dtype::float16 fake_attr_f16, phi::DataType fake_attr_dtype, const phi::Scalar& fake_attr_scalar, const phi::IntArray& fake_attr_int_array, @@ -64,7 +63,6 @@ void FakeDot(const Context& dev_ctx, std::cout << "fake_attr_float: " << fake_attr_float << std::endl; std::cout << "fake_attr_double: " << fake_attr_double << std::endl; std::cout << "fake_attr_int64: " << fake_attr_int64 << std::endl; - std::cout << "fake_attr_f16: " << fake_attr_f16 << std::endl; std::cout << "fake_attr_dtype: " << fake_attr_dtype << std::endl; std::cout << "fake_attr_int64_vec: " << fake_attr_int64_vec.size() << std::endl; @@ -78,7 +76,6 @@ void FakeDot(const Context& dev_ctx, assert(fake_attr_float == 2); assert(fake_attr_double == 3); assert(fake_attr_int64 == 4); - assert(fake_attr_f16 == phi::dtype::float16(5)); assert(fake_attr_dtype == phi::DataType::UINT32); assert(fake_attr_int64_vec.size() == 0); assert(fake_attr_int_vec.size() == 0); @@ -248,7 +245,6 @@ TEST(CustomKernel, custom_kernel_dot) { float fake_attr_float = 2.0; double fake_attr_double = 3.0; int64_t fake_attr_int64 = 4; - phi::dtype::float16 fake_attr_f16 = phi::dtype::float16(5); phi::DataType fake_attr_dtype = phi::DataType::UINT32; paddle::framework::LoDTensor tmp_tensor; tmp_tensor.mutable_data({1}, phi::TransToPhiPlace(backend)); @@ -262,7 +258,6 @@ TEST(CustomKernel, custom_kernel_dot) { kernel_context.EmplaceBackAttr(fake_attr_float); kernel_context.EmplaceBackAttr(fake_attr_double); kernel_context.EmplaceBackAttr(fake_attr_int64); - kernel_context.EmplaceBackAttr(fake_attr_f16); kernel_context.EmplaceBackAttr(fake_attr_dtype); kernel_context.EmplaceBackAttr(fake_attr_scalar); kernel_context.EmplaceBackAttr(fake_attr_int_array); diff --git a/paddle/phi/tests/core/test_kernel_factory.cc b/paddle/phi/tests/core/test_kernel_factory.cc index cb4b50f5b6c3d..490d4967eeba2 100644 --- a/paddle/phi/tests/core/test_kernel_factory.cc +++ b/paddle/phi/tests/core/test_kernel_factory.cc @@ -73,6 +73,67 @@ TEST(KernelRegistry, SetFP32Input) { EXPECT_EQ(output_defs.at(0).dtype, phi::DataType::FLOAT16); } +TEST(AttributeType, OStream) { + std::ostringstream oss; + oss << phi::AttributeType::UNDEFINED; + EXPECT_EQ(oss.str(), "Undefined"); + oss.str(""); + oss << phi::AttributeType::BOOL; + EXPECT_EQ(oss.str(), "bool"); + oss.str(""); + oss << phi::AttributeType::INT32; + EXPECT_EQ(oss.str(), "int"); + oss.str(""); + oss << phi::AttributeType::INT64; + EXPECT_EQ(oss.str(), "int64_t"); + oss.str(""); + oss << phi::AttributeType::FLOAT32; + EXPECT_EQ(oss.str(), "float"); + oss.str(""); + oss << phi::AttributeType::FLOAT64; + EXPECT_EQ(oss.str(), "double"); + oss.str(""); + oss << phi::AttributeType::STRING; + EXPECT_EQ(oss.str(), "string"); + oss.str(""); + oss << phi::AttributeType::BOOLS; + EXPECT_EQ(oss.str(), "vector"); + oss.str(""); + oss << phi::AttributeType::INT32S; + EXPECT_EQ(oss.str(), "vector"); + oss.str(""); + oss << phi::AttributeType::INT64S; + EXPECT_EQ(oss.str(), "vector"); + oss.str(""); + oss << phi::AttributeType::FLOAT32S; + EXPECT_EQ(oss.str(), "vector"); + oss.str(""); + oss << phi::AttributeType::FLOAT64S; + EXPECT_EQ(oss.str(), "vector"); + oss.str(""); + oss << phi::AttributeType::STRINGS; + EXPECT_EQ(oss.str(), "vector"); + oss.str(""); + oss << phi::AttributeType::SCALAR; + EXPECT_EQ(oss.str(), "Scalar"); + oss.str(""); + oss << phi::AttributeType::SCALARS; + EXPECT_EQ(oss.str(), "vector"); + oss.str(""); + oss << phi::AttributeType::INT_ARRAY; + EXPECT_EQ(oss.str(), "IntArray"); + oss.str(""); + oss << phi::AttributeType::DATA_TYPE; + EXPECT_EQ(oss.str(), "DataType"); + oss.str(""); + oss << phi::AttributeType::DATA_LAYOUT; + EXPECT_EQ(oss.str(), "DataLayout"); + oss.str(""); + oss << phi::AttributeType::PLACE; + EXPECT_EQ(oss.str(), "Place"); + oss.str(""); +} + } // namespace tests } // namespace phi diff --git a/paddle/phi/tests/core/test_meta_fn_utils.cc b/paddle/phi/tests/core/test_meta_fn_utils.cc index 028b9d23352c7..afdd3bc0d9ad0 100644 --- a/paddle/phi/tests/core/test_meta_fn_utils.cc +++ b/paddle/phi/tests/core/test_meta_fn_utils.cc @@ -60,32 +60,6 @@ TEST(MetaFnFactory, InferMetaFnExists) { EXPECT_EQ(dense_out1.dims()[1], dense_out2.dims()[1]); } -TEST(MetaFnFactory, CopyInferMetaFn) { - phi::DenseTensor dense_x; - dense_x.Resize({3, 4}); - - phi::MetaTensor meta_x(&dense_x); - phi::DenseTensor dense_out1; - phi::MetaTensor meta_out(&dense_out1); - phi::UnchangedInferMeta(meta_x, &meta_out); - - auto shared_meat_x = phi::MetaTensor(&dense_x); - phi::DenseTensor dense_out2; - auto shared_meta_out = phi::MetaTensor(&dense_out2); - - phi::InferMetaContext ctx; - ctx.EmplaceBackInput(shared_meat_x); - ctx.EmplaceBackAttr(Backend::CPU); - ctx.EmplaceBackAttr(false); - ctx.EmplaceBackOutput(shared_meta_out); - ctx.SetMetaConfig({/*is_runtime =*/true, /*is_run_mkldnn_kernel=*/false}); - phi::MetaFnFactory::Instance().Get("copy_to")(&ctx); - - EXPECT_EQ(dense_out1.dims().size(), dense_out2.dims().size()); - EXPECT_EQ(dense_out1.dims()[0], dense_out2.dims()[0]); - EXPECT_EQ(dense_out1.dims()[1], dense_out2.dims()[1]); -} - TEST(MetaFnFactory, SplitInferMetaFn) { phi::DenseTensor dense_x; dense_x.Resize({4, 10}); @@ -94,7 +68,7 @@ TEST(MetaFnFactory, SplitInferMetaFn) { phi::DenseTensor dense_out1; phi::DenseTensor dense_out2; - paddle::SmallVector out; + paddle::small_vector out; out.emplace_back(phi::MetaTensor(&dense_out1)); out.emplace_back(phi::MetaTensor(&dense_out2)); diff --git a/paddle/phi/tests/ops/test_op_signature.cc b/paddle/phi/tests/ops/test_op_signature.cc index 6acf3916a1866..4379dfd7cc4af 100644 --- a/paddle/phi/tests/ops/test_op_signature.cc +++ b/paddle/phi/tests/ops/test_op_signature.cc @@ -30,8 +30,8 @@ namespace tests { TEST(ARG_MAP, fill_constant) { TestArgumentMappingContext arg_case1( {"ShapeTensor", "ValueTensor"}, {}, {}, {}, {"Out"}); - auto signature1 = - OpUtilsMap::Instance().GetArgumentMappingFn("fill_constant")(arg_case1); + auto signature1 = (*OpUtilsMap::Instance().GetArgumentMappingFn( + "fill_constant"))(arg_case1); ASSERT_EQ(signature1.name, "full_sr"); TestArgumentMappingContext arg_case2( @@ -40,8 +40,8 @@ TEST(ARG_MAP, fill_constant) { {{"str_value", paddle::any{std::string{"10"}}}}, {}, {"Out"}); - auto signature2 = - OpUtilsMap::Instance().GetArgumentMappingFn("fill_constant")(arg_case2); + auto signature2 = (*OpUtilsMap::Instance().GetArgumentMappingFn( + "fill_constant"))(arg_case2); ASSERT_EQ(signature2.name, "full_sr"); TestArgumentMappingContext arg_case3( @@ -50,14 +50,14 @@ TEST(ARG_MAP, fill_constant) { {{"value", paddle::any{0}}, {"str_value", paddle::any{std::string{""}}}}, {}, {"Out"}); - auto signature3 = - OpUtilsMap::Instance().GetArgumentMappingFn("fill_constant")(arg_case3); + auto signature3 = (*OpUtilsMap::Instance().GetArgumentMappingFn( + "fill_constant"))(arg_case3); ASSERT_EQ(signature3.name, "full_sr"); TestArgumentMappingContext arg_case4( {"ShapeTensorList", "ValueTensor"}, {}, {}, {}, {"Out"}); - auto signature4 = - OpUtilsMap::Instance().GetArgumentMappingFn("fill_constant")(arg_case4); + auto signature4 = (*OpUtilsMap::Instance().GetArgumentMappingFn( + "fill_constant"))(arg_case4); ASSERT_EQ(signature4.name, "full_sr"); TestArgumentMappingContext arg_case5( @@ -66,8 +66,8 @@ TEST(ARG_MAP, fill_constant) { {{"str_value", paddle::any{std::string{"10"}}}}, {}, {"Out"}); - auto signature5 = - OpUtilsMap::Instance().GetArgumentMappingFn("fill_constant")(arg_case5); + auto signature5 = (*OpUtilsMap::Instance().GetArgumentMappingFn( + "fill_constant"))(arg_case5); ASSERT_EQ(signature5.name, "full_sr"); TestArgumentMappingContext arg_case6( @@ -76,8 +76,8 @@ TEST(ARG_MAP, fill_constant) { {{"value", paddle::any{0}}, {"str_value", paddle::any{std::string{""}}}}, {}, {"Out"}); - auto signature6 = - OpUtilsMap::Instance().GetArgumentMappingFn("fill_constant")(arg_case6); + auto signature6 = (*OpUtilsMap::Instance().GetArgumentMappingFn( + "fill_constant"))(arg_case6); ASSERT_EQ(signature6.name, "full_sr"); TestArgumentMappingContext arg_case7( @@ -86,8 +86,8 @@ TEST(ARG_MAP, fill_constant) { {{"shape", paddle::any{std::vector{2, 3}}}}, {}, {"Out"}); - auto signature7 = - OpUtilsMap::Instance().GetArgumentMappingFn("fill_constant")(arg_case7); + auto signature7 = (*OpUtilsMap::Instance().GetArgumentMappingFn( + "fill_constant"))(arg_case7); ASSERT_EQ(signature7.name, "full_sr"); TestArgumentMappingContext arg_case8( @@ -98,8 +98,8 @@ TEST(ARG_MAP, fill_constant) { {"str_value", paddle::any{std::string{""}}}}, {}, {"Out"}); - auto signature8 = - OpUtilsMap::Instance().GetArgumentMappingFn("fill_constant")(arg_case8); + auto signature8 = (*OpUtilsMap::Instance().GetArgumentMappingFn( + "fill_constant"))(arg_case8); ASSERT_EQ(signature8.name, "full_sr"); TestArgumentMappingContext arg_case9( @@ -109,8 +109,8 @@ TEST(ARG_MAP, fill_constant) { {"str_value", paddle::any{std::string{"10"}}}}, {}, {"Out"}); - auto signature9 = - OpUtilsMap::Instance().GetArgumentMappingFn("fill_constant")(arg_case9); + auto signature9 = (*OpUtilsMap::Instance().GetArgumentMappingFn( + "fill_constant"))(arg_case9); ASSERT_EQ(signature9.name, "full_sr"); } @@ -122,7 +122,8 @@ TEST(ARG_MAP, set_value) { {"Out"}, {}); ASSERT_EQ( - OpUtilsMap::Instance().GetArgumentMappingFn("set_value")(arg_case).name, + (*OpUtilsMap::Instance().GetArgumentMappingFn("set_value"))(arg_case) + .name, "set_value"); TestArgumentMappingContext arg_case1( @@ -132,7 +133,8 @@ TEST(ARG_MAP, set_value) { {"Out"}, {}); ASSERT_EQ( - OpUtilsMap::Instance().GetArgumentMappingFn("set_value")(arg_case1).name, + (*OpUtilsMap::Instance().GetArgumentMappingFn("set_value"))(arg_case1) + .name, "set_value"); TestArgumentMappingContext arg_case2( @@ -142,7 +144,8 @@ TEST(ARG_MAP, set_value) { {"Out"}, {}); ASSERT_EQ( - OpUtilsMap::Instance().GetArgumentMappingFn("set_value")(arg_case2).name, + (*OpUtilsMap::Instance().GetArgumentMappingFn("set_value"))(arg_case2) + .name, "set_value"); TestArgumentMappingContext arg_case3( @@ -152,7 +155,8 @@ TEST(ARG_MAP, set_value) { {"Out"}, {}); ASSERT_EQ( - OpUtilsMap::Instance().GetArgumentMappingFn("set_value")(arg_case3).name, + (*OpUtilsMap::Instance().GetArgumentMappingFn("set_value"))(arg_case3) + .name, "set_value"); TestArgumentMappingContext arg_case4( @@ -162,7 +166,8 @@ TEST(ARG_MAP, set_value) { {"Out"}, {}); ASSERT_EQ( - OpUtilsMap::Instance().GetArgumentMappingFn("set_value")(arg_case4).name, + (*OpUtilsMap::Instance().GetArgumentMappingFn("set_value"))(arg_case4) + .name, "set_value"); TestArgumentMappingContext arg_case5( @@ -172,7 +177,8 @@ TEST(ARG_MAP, set_value) { {"Out"}, {}); ASSERT_EQ( - OpUtilsMap::Instance().GetArgumentMappingFn("set_value")(arg_case5).name, + (*OpUtilsMap::Instance().GetArgumentMappingFn("set_value"))(arg_case5) + .name, "set_value_with_tensor"); TestArgumentMappingContext arg_case6( @@ -182,7 +188,8 @@ TEST(ARG_MAP, set_value) { {"Out"}, {}); ASSERT_EQ( - OpUtilsMap::Instance().GetArgumentMappingFn("set_value")(arg_case6).name, + (*OpUtilsMap::Instance().GetArgumentMappingFn("set_value"))(arg_case6) + .name, "set_value"); TestArgumentMappingContext arg_case7( @@ -192,7 +199,8 @@ TEST(ARG_MAP, set_value) { {"Out"}, {}); ASSERT_EQ( - OpUtilsMap::Instance().GetArgumentMappingFn("set_value")(arg_case7).name, + (*OpUtilsMap::Instance().GetArgumentMappingFn("set_value"))(arg_case7) + .name, "set_value"); TestArgumentMappingContext arg_case8( @@ -202,7 +210,8 @@ TEST(ARG_MAP, set_value) { {"Out"}, {}); ASSERT_EQ( - OpUtilsMap::Instance().GetArgumentMappingFn("set_value")(arg_case8).name, + (*OpUtilsMap::Instance().GetArgumentMappingFn("set_value"))(arg_case8) + .name, "set_value"); TestArgumentMappingContext arg_case9( @@ -212,7 +221,8 @@ TEST(ARG_MAP, set_value) { {"Out"}, {}); ASSERT_EQ( - OpUtilsMap::Instance().GetArgumentMappingFn("set_value")(arg_case9).name, + (*OpUtilsMap::Instance().GetArgumentMappingFn("set_value"))(arg_case9) + .name, "set_value"); TestArgumentMappingContext arg_case10( @@ -222,7 +232,8 @@ TEST(ARG_MAP, set_value) { {"Out"}, {}); ASSERT_EQ( - OpUtilsMap::Instance().GetArgumentMappingFn("set_value")(arg_case10).name, + (*OpUtilsMap::Instance().GetArgumentMappingFn("set_value"))(arg_case10) + .name, "set_value_with_tensor"); TestArgumentMappingContext arg_case11( @@ -232,7 +243,8 @@ TEST(ARG_MAP, set_value) { {"Out"}, {}); ASSERT_EQ( - OpUtilsMap::Instance().GetArgumentMappingFn("set_value")(arg_case11).name, + (*OpUtilsMap::Instance().GetArgumentMappingFn("set_value"))(arg_case11) + .name, "set_value"); TestArgumentMappingContext arg_case12( @@ -242,7 +254,8 @@ TEST(ARG_MAP, set_value) { {"Out"}, {}); ASSERT_EQ( - OpUtilsMap::Instance().GetArgumentMappingFn("set_value")(arg_case12).name, + (*OpUtilsMap::Instance().GetArgumentMappingFn("set_value"))(arg_case12) + .name, "set_value"); TestArgumentMappingContext arg_case13( @@ -252,7 +265,8 @@ TEST(ARG_MAP, set_value) { {"Out"}, {}); ASSERT_EQ( - OpUtilsMap::Instance().GetArgumentMappingFn("set_value")(arg_case13).name, + (*OpUtilsMap::Instance().GetArgumentMappingFn("set_value"))(arg_case13) + .name, "set_value"); TestArgumentMappingContext arg_case14( @@ -262,13 +276,15 @@ TEST(ARG_MAP, set_value) { {"Out"}, {}); ASSERT_EQ( - OpUtilsMap::Instance().GetArgumentMappingFn("set_value")(arg_case14).name, + (*OpUtilsMap::Instance().GetArgumentMappingFn("set_value"))(arg_case14) + .name, "set_value"); TestArgumentMappingContext arg_case15( {"Input", "StartsTensorList", "ValueTensor"}, {}, {}, {"Out"}, {}); ASSERT_EQ( - OpUtilsMap::Instance().GetArgumentMappingFn("set_value")(arg_case15).name, + (*OpUtilsMap::Instance().GetArgumentMappingFn("set_value"))(arg_case15) + .name, "set_value_with_tensor"); TestArgumentMappingContext arg_case16( @@ -278,7 +294,8 @@ TEST(ARG_MAP, set_value) { {"Out"}, {}); ASSERT_EQ( - OpUtilsMap::Instance().GetArgumentMappingFn("set_value")(arg_case16).name, + (*OpUtilsMap::Instance().GetArgumentMappingFn("set_value"))(arg_case16) + .name, "set_value"); TestArgumentMappingContext arg_case17( @@ -288,7 +305,8 @@ TEST(ARG_MAP, set_value) { {"Out"}, {}); ASSERT_EQ( - OpUtilsMap::Instance().GetArgumentMappingFn("set_value")(arg_case17).name, + (*OpUtilsMap::Instance().GetArgumentMappingFn("set_value"))(arg_case17) + .name, "set_value"); TestArgumentMappingContext arg_case18( @@ -298,7 +316,8 @@ TEST(ARG_MAP, set_value) { {"Out"}, {}); ASSERT_EQ( - OpUtilsMap::Instance().GetArgumentMappingFn("set_value")(arg_case18).name, + (*OpUtilsMap::Instance().GetArgumentMappingFn("set_value"))(arg_case18) + .name, "set_value"); TestArgumentMappingContext arg_case19( @@ -308,7 +327,8 @@ TEST(ARG_MAP, set_value) { {"Out"}, {}); ASSERT_EQ( - OpUtilsMap::Instance().GetArgumentMappingFn("set_value")(arg_case19).name, + (*OpUtilsMap::Instance().GetArgumentMappingFn("set_value"))(arg_case19) + .name, "set_value"); TestArgumentMappingContext arg_case20( @@ -318,7 +338,8 @@ TEST(ARG_MAP, set_value) { {"Out"}, {}); ASSERT_EQ( - OpUtilsMap::Instance().GetArgumentMappingFn("set_value")(arg_case20).name, + (*OpUtilsMap::Instance().GetArgumentMappingFn("set_value"))(arg_case20) + .name, "set_value"); TestArgumentMappingContext arg_case21( @@ -328,7 +349,8 @@ TEST(ARG_MAP, set_value) { {"Out"}, {}); ASSERT_EQ( - OpUtilsMap::Instance().GetArgumentMappingFn("set_value")(arg_case21).name, + (*OpUtilsMap::Instance().GetArgumentMappingFn("set_value"))(arg_case21) + .name, "set_value_with_tensor"); TestArgumentMappingContext arg_case22( @@ -338,7 +360,8 @@ TEST(ARG_MAP, set_value) { {"Out"}, {}); ASSERT_EQ( - OpUtilsMap::Instance().GetArgumentMappingFn("set_value")(arg_case22).name, + (*OpUtilsMap::Instance().GetArgumentMappingFn("set_value"))(arg_case22) + .name, "set_value"); TestArgumentMappingContext arg_case23( @@ -348,7 +371,8 @@ TEST(ARG_MAP, set_value) { {"Out"}, {}); ASSERT_EQ( - OpUtilsMap::Instance().GetArgumentMappingFn("set_value")(arg_case23).name, + (*OpUtilsMap::Instance().GetArgumentMappingFn("set_value"))(arg_case23) + .name, "set_value"); TestArgumentMappingContext arg_case24( @@ -358,7 +382,8 @@ TEST(ARG_MAP, set_value) { {"Out"}, {}); ASSERT_EQ( - OpUtilsMap::Instance().GetArgumentMappingFn("set_value")(arg_case24).name, + (*OpUtilsMap::Instance().GetArgumentMappingFn("set_value"))(arg_case24) + .name, "set_value"); TestArgumentMappingContext arg_case25( @@ -368,13 +393,15 @@ TEST(ARG_MAP, set_value) { {"Out"}, {}); ASSERT_EQ( - OpUtilsMap::Instance().GetArgumentMappingFn("set_value")(arg_case25).name, + (*OpUtilsMap::Instance().GetArgumentMappingFn("set_value"))(arg_case25) + .name, "set_value"); TestArgumentMappingContext arg_case26( {"Input", "EndsTensorList", "ValueTensor"}, {}, {}, {"Out"}, {}); ASSERT_EQ( - OpUtilsMap::Instance().GetArgumentMappingFn("set_value")(arg_case26).name, + (*OpUtilsMap::Instance().GetArgumentMappingFn("set_value"))(arg_case26) + .name, "set_value_with_tensor"); TestArgumentMappingContext arg_case27( @@ -384,7 +411,8 @@ TEST(ARG_MAP, set_value) { {"Out"}, {}); ASSERT_EQ( - OpUtilsMap::Instance().GetArgumentMappingFn("set_value")(arg_case27).name, + (*OpUtilsMap::Instance().GetArgumentMappingFn("set_value"))(arg_case27) + .name, "set_value"); TestArgumentMappingContext arg_case28( @@ -394,7 +422,8 @@ TEST(ARG_MAP, set_value) { {"Out"}, {}); ASSERT_EQ( - OpUtilsMap::Instance().GetArgumentMappingFn("set_value")(arg_case28).name, + (*OpUtilsMap::Instance().GetArgumentMappingFn("set_value"))(arg_case28) + .name, "set_value"); TestArgumentMappingContext arg_case29( @@ -404,7 +433,8 @@ TEST(ARG_MAP, set_value) { {"Out"}, {}); ASSERT_EQ( - OpUtilsMap::Instance().GetArgumentMappingFn("set_value")(arg_case29).name, + (*OpUtilsMap::Instance().GetArgumentMappingFn("set_value"))(arg_case29) + .name, "set_value"); TestArgumentMappingContext arg_case30( @@ -414,7 +444,8 @@ TEST(ARG_MAP, set_value) { {"Out"}, {}); ASSERT_EQ( - OpUtilsMap::Instance().GetArgumentMappingFn("set_value")(arg_case30).name, + (*OpUtilsMap::Instance().GetArgumentMappingFn("set_value"))(arg_case30) + .name, "set_value"); TestArgumentMappingContext arg_case31( @@ -424,13 +455,15 @@ TEST(ARG_MAP, set_value) { {"Out"}, {}); ASSERT_EQ( - OpUtilsMap::Instance().GetArgumentMappingFn("set_value")(arg_case31).name, + (*OpUtilsMap::Instance().GetArgumentMappingFn("set_value"))(arg_case31) + .name, "set_value"); TestArgumentMappingContext arg_case32( {"Input", "StepsTensorList", "ValueTensor"}, {}, {}, {"Out"}, {}); ASSERT_EQ( - OpUtilsMap::Instance().GetArgumentMappingFn("set_value")(arg_case32).name, + (*OpUtilsMap::Instance().GetArgumentMappingFn("set_value"))(arg_case32) + .name, "set_value_with_tensor"); TestArgumentMappingContext arg_case33( @@ -440,7 +473,8 @@ TEST(ARG_MAP, set_value) { {"Out"}, {}); ASSERT_EQ( - OpUtilsMap::Instance().GetArgumentMappingFn("set_value")(arg_case33).name, + (*OpUtilsMap::Instance().GetArgumentMappingFn("set_value"))(arg_case33) + .name, "set_value"); TestArgumentMappingContext arg_case34( @@ -450,7 +484,8 @@ TEST(ARG_MAP, set_value) { {"Out"}, {}); ASSERT_EQ( - OpUtilsMap::Instance().GetArgumentMappingFn("set_value")(arg_case34).name, + (*OpUtilsMap::Instance().GetArgumentMappingFn("set_value"))(arg_case34) + .name, "set_value"); TestArgumentMappingContext arg_case35( @@ -460,7 +495,8 @@ TEST(ARG_MAP, set_value) { {"Out"}, {}); ASSERT_EQ( - OpUtilsMap::Instance().GetArgumentMappingFn("set_value")(arg_case35).name, + (*OpUtilsMap::Instance().GetArgumentMappingFn("set_value"))(arg_case35) + .name, "set_value"); TestArgumentMappingContext arg_case36( @@ -470,7 +506,8 @@ TEST(ARG_MAP, set_value) { {"Out"}, {}); ASSERT_EQ( - OpUtilsMap::Instance().GetArgumentMappingFn("set_value")(arg_case36).name, + (*OpUtilsMap::Instance().GetArgumentMappingFn("set_value"))(arg_case36) + .name, "set_value"); TestArgumentMappingContext arg_case37( @@ -480,7 +517,8 @@ TEST(ARG_MAP, set_value) { {"Out"}, {}); ASSERT_EQ( - OpUtilsMap::Instance().GetArgumentMappingFn("set_value")(arg_case37).name, + (*OpUtilsMap::Instance().GetArgumentMappingFn("set_value"))(arg_case37) + .name, "set_value"); } @@ -491,10 +529,10 @@ TEST(ARG_MAP, set_value_grad) { {}, {"Input@GRAD", "ValueTensor@GRAD"}, {}); - ASSERT_EQ(OpUtilsMap::Instance() - .GetArgumentMappingFn("set_value_grad")(arg_case) - .name, - "set_value_grad"); + ASSERT_EQ( + (*OpUtilsMap::Instance().GetArgumentMappingFn("set_value_grad"))(arg_case) + .name, + "set_value_grad"); TestArgumentMappingContext arg_case1( {"Out@GRAD", "StartsTensorList", "StepsTensorList"}, @@ -502,8 +540,8 @@ TEST(ARG_MAP, set_value_grad) { {}, {"Input@GRAD", "ValueTensor@GRAD"}, {}); - ASSERT_EQ(OpUtilsMap::Instance() - .GetArgumentMappingFn("set_value_grad")(arg_case1) + ASSERT_EQ((*OpUtilsMap::Instance().GetArgumentMappingFn("set_value_grad"))( + arg_case1) .name, "set_value_grad"); @@ -512,8 +550,8 @@ TEST(ARG_MAP, set_value_grad) { {}, {"Input@GRAD", "ValueTensor@GRAD"}, {}); - ASSERT_EQ(OpUtilsMap::Instance() - .GetArgumentMappingFn("set_value_grad")(arg_case2) + ASSERT_EQ((*OpUtilsMap::Instance().GetArgumentMappingFn("set_value_grad"))( + arg_case2) .name, "set_value_grad"); @@ -523,8 +561,8 @@ TEST(ARG_MAP, set_value_grad) { {}, {"Input@GRAD", "ValueTensor@GRAD"}, {}); - ASSERT_EQ(OpUtilsMap::Instance() - .GetArgumentMappingFn("set_value_grad")(arg_case3) + ASSERT_EQ((*OpUtilsMap::Instance().GetArgumentMappingFn("set_value_grad"))( + arg_case3) .name, "set_value_grad"); @@ -533,8 +571,8 @@ TEST(ARG_MAP, set_value_grad) { {}, {"Input@GRAD", "ValueTensor@GRAD"}, {}); - ASSERT_EQ(OpUtilsMap::Instance() - .GetArgumentMappingFn("set_value_grad")(arg_case4) + ASSERT_EQ((*OpUtilsMap::Instance().GetArgumentMappingFn("set_value_grad"))( + arg_case4) .name, "set_value_grad"); @@ -543,8 +581,8 @@ TEST(ARG_MAP, set_value_grad) { {}, {"Input@GRAD", "ValueTensor@GRAD"}, {}); - ASSERT_EQ(OpUtilsMap::Instance() - .GetArgumentMappingFn("set_value_grad")(arg_case5) + ASSERT_EQ((*OpUtilsMap::Instance().GetArgumentMappingFn("set_value_grad"))( + arg_case5) .name, "set_value_grad"); } @@ -558,10 +596,9 @@ TEST(ARG_MAP, allclose) { {"Out"}, {}); auto signature1 = - OpUtilsMap::Instance().GetArgumentMappingFn("allclose")(arg_case1); + (*OpUtilsMap::Instance().GetArgumentMappingFn("allclose"))(arg_case1); ASSERT_EQ(signature1.name, "allclose"); - auto attr_names1 = std::get<1>(signature1.args); - ASSERT_EQ(attr_names1[0], "Rtol"); + ASSERT_EQ(signature1.attr_names[0], "Rtol"); TestArgumentMappingContext arg_case2( {"Input", "Other", "Atol"}, @@ -571,27 +608,26 @@ TEST(ARG_MAP, allclose) { {"Out"}, {}); auto signature2 = - OpUtilsMap::Instance().GetArgumentMappingFn("allclose")(arg_case2); + (*OpUtilsMap::Instance().GetArgumentMappingFn("allclose"))(arg_case2); ASSERT_EQ(signature2.name, "allclose"); - auto attr_names2 = std::get<1>(signature2.args); - ASSERT_EQ(attr_names2[1], "Atol"); + ASSERT_EQ(signature2.attr_names[1], "Atol"); } TEST(ARG_MAP, reshape) { TestArgumentMappingContext arg_case1({"X", "ShapeTensor"}, {}, {}, {"Out"}); auto signature1 = - OpUtilsMap::Instance().GetArgumentMappingFn("reshape2")(arg_case1); + (*OpUtilsMap::Instance().GetArgumentMappingFn("reshape2"))(arg_case1); ASSERT_EQ(signature1.name, "reshape"); TestArgumentMappingContext arg_case2({"X", "Shape"}, {}, {}, {"Out"}); auto signature2 = - OpUtilsMap::Instance().GetArgumentMappingFn("reshape2")(arg_case2); + (*OpUtilsMap::Instance().GetArgumentMappingFn("reshape2"))(arg_case2); ASSERT_EQ(signature2.name, "reshape"); TestArgumentMappingContext arg_case3( {"X"}, {}, {{"shape", paddle::any(std::vector({1, 2}))}}, {"Out"}); auto signature3 = - OpUtilsMap::Instance().GetArgumentMappingFn("reshape2")(arg_case3); + (*OpUtilsMap::Instance().GetArgumentMappingFn("reshape2"))(arg_case3); ASSERT_EQ(signature3.name, "reshape"); } diff --git a/paddle/phi/tests/ops/test_op_signature.h b/paddle/phi/tests/ops/test_op_signature.h index 4a84793527ea7..1535f40b70072 100644 --- a/paddle/phi/tests/ops/test_op_signature.h +++ b/paddle/phi/tests/ops/test_op_signature.h @@ -68,6 +68,10 @@ class TestArgumentMappingContext : public phi::ArgumentMappingContext { return dense_tensor_inputs.count(name) > 0; } + bool IsDenseTensorInputs(const std::string& name) const override { + return dense_tensor_inputs.count(name) > 0; + } + bool IsSelectedRowsInput(const std::string& name) const override { return selected_rows_inputs.count(name) > 0; } diff --git a/paddle/scripts/paddle_build.bat b/paddle/scripts/paddle_build.bat index a7a2592f971c5..f4a09436d86ce 100644 --- a/paddle/scripts/paddle_build.bat +++ b/paddle/scripts/paddle_build.bat @@ -86,6 +86,10 @@ if not defined NEW_RELEASE_JIT set NEW_RELEASE_JIT=OFF set task_name=%1 set UPLOAD_TP_FILE=OFF +rem ------initialize set git config------ +git config --global core.longpaths true + + rem ------initialize the python environment------ set PYTHON_EXECUTABLE=%PYTHON_ROOT%\python.exe set PATH=%PYTHON_ROOT%\Scripts;%PYTHON_ROOT%;%PATH% @@ -255,6 +259,7 @@ set MSVC_STATIC_CRT=ON set ON_INFER=ON set WITH_TENSORRT=ON set WITH_INFERENCE_API_TEST=ON +set WITH_ONNXRUNTIME=ON call :cmake || goto cmake_error call :build || goto build_error diff --git a/paddle/scripts/paddle_build.sh b/paddle/scripts/paddle_build.sh index 2e2efa65d7007..5f0a70dc0e69f 100755 --- a/paddle/scripts/paddle_build.sh +++ b/paddle/scripts/paddle_build.sh @@ -132,6 +132,18 @@ function cmake_base() { else exit 1 fi + elif [ "$1" == "cp310-cp310" ]; then + if [ -d "/Library/Frameworks/Python.framework/Versions/3.10" ]; then + export LD_LIBRARY_PATH=/Library/Frameworks/Python.framework/Versions/3.10/lib/ + export DYLD_LIBRARY_PATH=/Library/Frameworks/Python.framework/Versions/3.10/lib/ + export PATH=/Library/Frameworks/Python.framework/Versions/3.10/bin/:${PATH} + PYTHON_FLAGS="-DPYTHON_EXECUTABLE:FILEPATH=/Library/Frameworks/Python.framework/Versions/3.10/bin/python3 + -DPYTHON_INCLUDE_DIR:PATH=/Library/Frameworks/Python.framework/Versions/3.10/include/python3.10/ + -DPYTHON_LIBRARY:FILEPATH=/Library/Frameworks/Python.framework/Versions/3.10/lib/libpython3.10.dylib" + pip3.10 install --user -r ${PADDLE_ROOT}/python/requirements.txt + else + exit 1 + fi fi else if [ "$1" != "" ]; then @@ -164,6 +176,13 @@ function cmake_base() { -DPYTHON_INCLUDE_DIR:PATH=/opt/_internal/cpython-3.9.0/include/python3.9 -DPYTHON_LIBRARIES:FILEPATH=/opt/_internal/cpython-3.9.0/lib/libpython3.so" pip3.9 install -r ${PADDLE_ROOT}/python/requirements.txt + elif [ "$1" == "cp310-cp310" ]; then + export LD_LIBRARY_PATH=/opt/_internal/cpython-3.10.0/lib/:${LD_LIBRARY_PATH} + export PATH=/opt/_internal/cpython-3.10.0/bin/:${PATH} + export PYTHON_FLAGS="-DPYTHON_EXECUTABLE:FILEPATH=/opt/_internal/cpython-3.10.0/bin/python3.10 + -DPYTHON_INCLUDE_DIR:PATH=/opt/_internal/cpython-3.10.0/include/python3.10 + -DPYTHON_LIBRARIES:FILEPATH=/opt/_internal/cpython-3.10.0/lib/libpython3.so" + pip3.10 install -r ${PADDLE_ROOT}/python/requirements.txt elif [ "$1" == "conda-python3.7" ]; then export LD_LIBRARY_PATH=/opt/conda/lib/:${LD_LIBRARY_PATH} export PATH=/opt/conda/bin/:${PATH} @@ -612,6 +631,8 @@ EOF pip3.8 uninstall -y paddlepaddle elif [ "$1" == "cp39-cp39" ]; then pip3.9 uninstall -y paddlepaddle + elif [ "$1" == "cp310-cp310" ]; then + pip3.10 uninstall -y paddlepaddle fi set -ex @@ -627,6 +648,9 @@ EOF elif [ "$1" == "cp39-cp39" ]; then pip3.9 install --user ${INSTALL_PREFIX:-/paddle/build}/opt/paddle/share/wheels/*.whl pip3.9 install --user hypothesis + elif [ "$1" == "cp310-cp310" ]; then + pip3.10 install --user ${INSTALL_PREFIX:-/paddle/build}/opt/paddle/share/wheels/*.whl + pip3.10 install --user hypothesis fi tmpfile_rand=`date +%s%N` tmpfile=$tmp_dir/$tmpfile_rand @@ -728,6 +752,8 @@ function run_linux_cpu_test() { pip install hypothesis pip install ${PADDLE_ROOT}/build/python/dist/*whl cp ${PADDLE_ROOT}/build/python/paddle/fluid/tests/unittests/op_test.py ${PADDLE_ROOT}/build/python + cp ${PADDLE_ROOT}/build/python/paddle/fluid/tests/unittests/testsuite.py ${PADDLE_ROOT}/build/python + cp -r ${PADDLE_ROOT}/build/python/paddle/fluid/tests/unittests/white_list ${PADDLE_ROOT}/build/python ut_total_startTime_s=`date +%s` if [ ${WITH_TESTING:-ON} == "ON" ] ; then cat <> ${PADDLE_ROOT}/build/Dockerfile < /dev/null && \ + make -j8 > /dev/null && make altinstall > /dev/null && cd ../ && rm Python-3.10.0.tgz + RUN apt-get install -y libgtk2.0-dev dmidecode python3-tk && ldconfig && \ + wget ${ref_web}/${ref_paddle310} && pip3.10 install ${ref_paddle310_whl}; apt-get install -f -y && \ + apt-get clean -y && \ + rm -f ${ref_paddle310} && \ + ldconfig EOF cat >> ${PADDLE_ROOT}/build/Dockerfile < -class ArrayRef { +class array_ref { public: using iterator = const T *; using const_iterator = const T *; @@ -59,70 +61,81 @@ class ArrayRef { /// @name Constructors /// @{ - /// Construct an empty ArrayRef. - /*implicit*/ ArrayRef() = default; + /// Construct an empty array_ref. + /*implicit*/ array_ref() = default; - /// Construct an empty ArrayRef from None. - /*implicit*/ ArrayRef(none_t) {} + /// Construct an empty array_ref from None. + /*implicit*/ array_ref(none_t) {} - /// Construct an ArrayRef from a single element. - /*implicit*/ ArrayRef(const T &OneElt) : Data(&OneElt), Length(1) {} + /// Construct an array_ref from a single element. + /*implicit*/ array_ref(const T &OneElt) : Data(&OneElt), Length(1) {} - /// Construct an ArrayRef from a pointer and length. - /*implicit*/ ArrayRef(const T *data, size_t length) + /// Construct an array_ref from a pointer and length. + /*implicit*/ array_ref(const T *data, size_t length) : Data(data), Length(length) {} - /// Construct an ArrayRef from a range. - ArrayRef(const T *begin, const T *end) : Data(begin), Length(end - begin) {} + /// Construct an array_ref from a range. + array_ref(const T *begin, const T *end) : Data(begin), Length(end - begin) {} - /// Construct an ArrayRef from a SmallVector. This is templated in order to - /// avoid instantiating SmallVectorTemplateCommon whenever we - /// copy-construct an ArrayRef. + /// Construct an array_ref from a small_vector. This is templated in order to + /// avoid instantiating small_vector_template_common whenever we + /// copy-construct an array_ref. template - /*implicit*/ ArrayRef(const SmallVectorTemplateCommon &Vec) + /*implicit*/ array_ref(const small_vector_template_common &Vec) : Data(Vec.data()), Length(Vec.size()) {} - /// Construct an ArrayRef from a std::vector. + /// Construct an array_ref from a std::vector. template - /*implicit*/ ArrayRef(const std::vector &Vec) + /*implicit*/ array_ref(const std::vector &Vec) : Data(Vec.data()), Length(Vec.size()) {} - /// Construct an ArrayRef from a std::array + /// Construct an array_ref from a std::array template - /*implicit*/ constexpr ArrayRef(const std::array &Arr) + /*implicit*/ constexpr array_ref(const std::array &Arr) : Data(Arr.data()), Length(N) {} - /// Construct an ArrayRef from a C array. + /// Construct an array_ref from a C array. template - /*implicit*/ constexpr ArrayRef(const T (&Arr)[N]) : Data(Arr), Length(N) {} - - /// Construct an ArrayRef from a std::initializer_list. - /*implicit*/ ArrayRef(const std::initializer_list &Vec) + /*implicit*/ constexpr array_ref(const T (&Arr)[N]) : Data(Arr), Length(N) {} + +/// Construct an array_ref from a std::initializer_list. +#if defined(__GNUC__) && !defined(__clang__) && __GNUC__ >= 9 +// Disable gcc's warning in this constructor as it generates an enormous +// amount +// of messages. Anyone using array_ref should already be aware of the fact that +// it does not do lifetime extension. +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Winit-list-lifetime" +#endif + /*implicit*/ array_ref(const std::initializer_list &Vec) : Data(Vec.begin() == Vec.end() ? (T *)nullptr : Vec.begin()), Length(Vec.size()) {} +#if defined(__GNUC__) && !defined(__clang__) && __GNUC__ >= 9 +#pragma GCC diagnostic pop +#endif - /// Construct an ArrayRef from ArrayRef. This uses SFINAE to + /// Construct an array_ref from array_ref. This uses SFINAE to /// ensure that only ArrayRefs of pointers can be converted. template - ArrayRef(const ArrayRef &A, - std::enable_if_t::value> - * = nullptr) + array_ref(const array_ref &A, + std::enable_if_t::value> + * = nullptr) : Data(A.data()), Length(A.size()) {} - /// Construct an ArrayRef from a SmallVector. This is - /// templated in order to avoid instantiating SmallVectorTemplateCommon - /// whenever we copy-construct an ArrayRef. + /// Construct an array_ref from a small_vector. This is + /// templated in order to avoid instantiating small_vector_template_common + /// whenever we copy-construct an array_ref. template - /*implicit*/ ArrayRef( - const SmallVectorTemplateCommon &Vec, + /*implicit*/ array_ref( + const small_vector_template_common &Vec, std::enable_if_t::value> * = nullptr) : Data(Vec.data()), Length(Vec.size()) {} - /// Construct an ArrayRef from std::vector. This uses SFINAE + /// Construct an array_ref from std::vector. This uses SFINAE /// to ensure that only vectors of pointers can be converted. template - ArrayRef( + array_ref( const std::vector &Vec, std::enable_if_t::value> * = 0) : Data(Vec.data()), Length(Vec.size()) {} @@ -157,50 +170,50 @@ class ArrayRef { return Data[Length - 1]; } - // copy - Allocate copy in Allocator and return ArrayRef to it. + // copy - Allocate copy in Allocator and return array_ref to it. template - ArrayRef copy(Allocator &A) { + array_ref copy(Allocator &A) { T *Buff = A.template Allocate(Length); std::uninitialized_copy(begin(), end(), Buff); - return ArrayRef(Buff, Length); + return array_ref(Buff, Length); } /// equals - Check for element-wise equality. - bool equals(ArrayRef RHS) const { + bool equals(array_ref RHS) const { if (Length != RHS.Length) return false; return std::equal(begin(), end(), RHS.begin()); } /// slice(n, m) - Chop off the first N elements of the array, and keep M /// elements in the array. - ArrayRef slice(size_t N, size_t M) const { + array_ref slice(size_t N, size_t M) const { assert(N + M <= size() && "Invalid specifier"); - return ArrayRef(data() + N, M); + return array_ref(data() + N, M); } /// slice(n) - Chop off the first N elements of the array. - ArrayRef slice(size_t N) const { return slice(N, size() - N); } + array_ref slice(size_t N) const { return slice(N, size() - N); } /// Drop the first \p N elements of the array. - ArrayRef drop_front(size_t N = 1) const { + array_ref drop_front(size_t N = 1) const { assert(size() >= N && "Dropping more elements than exist"); return slice(N, size() - N); } /// Drop the last \p N elements of the array. - ArrayRef drop_back(size_t N = 1) const { + array_ref drop_back(size_t N = 1) const { assert(size() >= N && "Dropping more elements than exist"); return slice(0, size() - N); } /// Return a copy of *this with only the first \p N elements. - ArrayRef take_front(size_t N = 1) const { + array_ref take_front(size_t N = 1) const { if (N >= size()) return *this; return drop_back(size() - N); } /// Return a copy of *this with only the last \p N elements. - ArrayRef take_back(size_t N = 1) const { + array_ref take_back(size_t N = 1) const { if (N >= size()) return *this; return drop_front(size() - N); } @@ -218,7 +231,7 @@ class ArrayRef { /// The declaration here is extra complicated so that "arrayRef = {}" /// continues to select the move assignment operator. template - std::enable_if_t::value, ArrayRef> &operator=( + std::enable_if_t::value, array_ref> &operator=( U &&Temporary) = delete; /// Disallow accidental assignment from a temporary. @@ -226,7 +239,7 @@ class ArrayRef { /// The declaration here is extra complicated so that "arrayRef = {}" /// continues to select the move assignment operator. template - std::enable_if_t::value, ArrayRef> &operator=( + std::enable_if_t::value, array_ref> &operator=( std::initializer_list) = delete; /// @} @@ -244,90 +257,90 @@ class ArrayRef { /// @} }; -/// @name ArrayRef Convenience constructors +/// @name array_ref Convenience constructors /// @{ -/// Construct an ArrayRef from a single element. +/// Construct an array_ref from a single element. template -ArrayRef makeArrayRef(const T &OneElt) { +array_ref make_array_ref(const T &OneElt) { return OneElt; } -/// Construct an ArrayRef from a pointer and length. +/// Construct an array_ref from a pointer and length. template -ArrayRef makeArrayRef(const T *data, size_t length) { - return ArrayRef(data, length); +array_ref make_array_ref(const T *data, size_t length) { + return array_ref(data, length); } -/// Construct an ArrayRef from a range. +/// Construct an array_ref from a range. template -ArrayRef makeArrayRef(const T *begin, const T *end) { - return ArrayRef(begin, end); +array_ref make_array_ref(const T *begin, const T *end) { + return array_ref(begin, end); } -/// Construct an ArrayRef from a SmallVector. +/// Construct an array_ref from a small_vector. template -ArrayRef makeArrayRef(const SmallVectorImpl &Vec) { +array_ref make_array_ref(const small_vector_impl &Vec) { return Vec; } -/// Construct an ArrayRef from a SmallVector. +/// Construct an array_ref from a small_vector. template -ArrayRef makeArrayRef(const SmallVector &Vec) { +array_ref make_array_ref(const small_vector &Vec) { return Vec; } -/// Construct an ArrayRef from a std::vector. +/// Construct an array_ref from a std::vector. template -ArrayRef makeArrayRef(const std::vector &Vec) { +array_ref make_array_ref(const std::vector &Vec) { return Vec; } -/// Construct an ArrayRef from a std::array. +/// Construct an array_ref from a std::array. template -ArrayRef makeArrayRef(const std::array &Arr) { +array_ref make_array_ref(const std::array &Arr) { return Arr; } -/// Construct an ArrayRef from an ArrayRef (no-op) (const) +/// Construct an array_ref from an array_ref (no-op) (const) template -ArrayRef makeArrayRef(const ArrayRef &Vec) { +array_ref make_array_ref(const array_ref &Vec) { return Vec; } -/// Construct an ArrayRef from an ArrayRef (no-op) +/// Construct an array_ref from an array_ref (no-op) template -ArrayRef &makeArrayRef(ArrayRef &Vec) { +array_ref &make_array_ref(array_ref &Vec) { return Vec; } -/// Construct an ArrayRef from a C array. +/// Construct an array_ref from a C array. template -ArrayRef makeArrayRef(const T (&Arr)[N]) { - return ArrayRef(Arr); +array_ref make_array_ref(const T (&Arr)[N]) { + return array_ref(Arr); } /// @} -/// @name ArrayRef Comparison Operators +/// @name array_ref Comparison Operators /// @{ template -inline bool operator==(ArrayRef LHS, ArrayRef RHS) { +inline bool operator==(array_ref LHS, array_ref RHS) { return LHS.equals(RHS); } template -inline bool operator==(SmallVectorImpl &LHS, ArrayRef RHS) { - return ArrayRef(LHS).equals(RHS); +inline bool operator==(small_vector_impl &LHS, array_ref RHS) { + return array_ref(LHS).equals(RHS); } template -inline bool operator!=(ArrayRef LHS, ArrayRef RHS) { +inline bool operator!=(array_ref LHS, array_ref RHS) { return !(LHS == RHS); } template -inline bool operator!=(SmallVectorImpl &LHS, ArrayRef RHS) { +inline bool operator!=(small_vector_impl &LHS, array_ref RHS) { return !(LHS == RHS); } diff --git a/paddle/utils/array_ref_test.cc b/paddle/utils/array_ref_test.cc index 33a09c499246d..cc4e88a5ee351 100644 --- a/paddle/utils/array_ref_test.cc +++ b/paddle/utils/array_ref_test.cc @@ -21,53 +21,53 @@ #include "gtest/gtest.h" TEST(array_ref, array_ref) { - paddle::ArrayRef a; + paddle::array_ref a; CHECK_EQ(a.size(), size_t(0)); CHECK_EQ(a.data(), static_cast(nullptr)); - paddle::ArrayRef b(paddle::none); + paddle::array_ref b(paddle::none); CHECK_EQ(b.size(), size_t(0)); CHECK_EQ(b.data(), static_cast(nullptr)); int v = 1; - paddle::ArrayRef c(v); + paddle::array_ref c(v); CHECK_EQ(c.size(), size_t(1)); CHECK_EQ(c.data(), &v); - CHECK_EQ(c.equals(paddle::makeArrayRef(v)), true); + CHECK_EQ(c.equals(paddle::make_array_ref(v)), true); int v1[5] = {1, 2, 3, 4, 5}; - paddle::ArrayRef d(v1, 5); + paddle::array_ref d(v1, 5); CHECK_EQ(d.size(), size_t(5)); CHECK_EQ(d.data(), v1); - CHECK_EQ(d.equals(paddle::makeArrayRef(v1, 5)), true); + CHECK_EQ(d.equals(paddle::make_array_ref(v1, 5)), true); - paddle::ArrayRef e(&v1[0], &v1[4]); + paddle::array_ref e(&v1[0], &v1[4]); CHECK_EQ(e.size(), size_t(4)); CHECK_EQ(e.data(), v1); - CHECK_EQ(e.equals(paddle::makeArrayRef(&v1[0], &v1[4])), true); + CHECK_EQ(e.equals(paddle::make_array_ref(&v1[0], &v1[4])), true); - paddle::SmallVector small_vector{1, 2, 3}; - paddle::ArrayRef f(small_vector); + paddle::small_vector small_vector{1, 2, 3}; + paddle::array_ref f(small_vector); CHECK_EQ(f.size(), size_t(3)); CHECK_EQ(f.data(), small_vector.data()); - CHECK_EQ(f.equals(paddle::makeArrayRef(small_vector)), true); + CHECK_EQ(f.equals(paddle::make_array_ref(small_vector)), true); std::vector vector{1, 2, 3}; - paddle::ArrayRef g(vector); + paddle::array_ref g(vector); CHECK_EQ(g.size(), size_t(3)); CHECK_EQ(g.data(), vector.data()); - CHECK_EQ(g.equals(paddle::makeArrayRef(vector)), true); + CHECK_EQ(g.equals(paddle::make_array_ref(vector)), true); std::initializer_list list = {1, 2, 3}; - paddle::ArrayRef h(list); + paddle::array_ref h(list); CHECK_EQ(h.size(), size_t(3)); CHECK_EQ(h.data(), list.begin()); - paddle::ArrayRef i(h); + paddle::array_ref i(h); CHECK_EQ(i.size(), size_t(3)); CHECK_EQ(i.data(), list.begin()); CHECK_EQ(i.equals(h), true); - CHECK_EQ(i.equals(paddle::makeArrayRef(h)), true); + CHECK_EQ(i.equals(paddle::make_array_ref(h)), true); auto slice = i.slice(1, 2); CHECK_EQ(slice.size(), size_t(2)); @@ -78,7 +78,7 @@ TEST(array_ref, array_ref) { CHECK_EQ(drop.size(), size_t(1)); CHECK_EQ(drop[0], 3); - paddle::ArrayRef nums = {1, 2, 3, 4, 5, 6, 7, 8}; + paddle::array_ref nums = {1, 2, 3, 4, 5, 6, 7, 8}; auto front = nums.take_front(3); CHECK_EQ(front.size(), size_t(3)); for (size_t i = 0; i < 3; ++i) { diff --git a/paddle/utils/small_vector.h b/paddle/utils/small_vector.h index 14cb8f410f460..27db9ae18822a 100644 --- a/paddle/utils/small_vector.h +++ b/paddle/utils/small_vector.h @@ -5,6 +5,7 @@ // 3. add at(index) method for small vector // 4. wrap the call to max and min with parenthesis to prevent the macro // expansion to fix the build error on windows platform +// 5. change SmallVector to small_vector to unify naming style of utils //===- llvm/ADT/SmallVector.h - 'Normally small' vectors --------*- C++ -*-===// // @@ -79,13 +80,13 @@ iterator_range make_range(std::pair p) { /// This is all the stuff common to all SmallVectors. /// /// The template parameter specifies the type which should be used to hold the -/// Size and Capacity of the SmallVector, so it can be adjusted. -/// Using 32 bit size is desirable to shrink the size of the SmallVector. -/// Using 64 bit size is desirable for cases like SmallVector, where a +/// Size and Capacity of the small_vector, so it can be adjusted. +/// Using 32 bit size is desirable to shrink the size of the small_vector. +/// Using 64 bit size is desirable for cases like small_vector, where a /// 32 bit size would limit the vector to ~4GB. SmallVectors are used for /// buffering bitcode output - which can exceed 4GB. template -class SmallVectorBase { +class small_vector_base { protected: void *BeginX; Size_T Size = 0, Capacity; @@ -95,8 +96,8 @@ class SmallVectorBase { return (std::numeric_limits::max)(); } - SmallVectorBase() = delete; - SmallVectorBase(void *FirstEl, size_t TotalCapacity) + small_vector_base() = delete; + small_vector_base(void *FirstEl, size_t TotalCapacity) : BeginX(FirstEl), Capacity(TotalCapacity) {} /// This is a helper for \a grow() that's out of line to reduce code @@ -139,22 +140,23 @@ using SmallVectorSizeType = /// Figure out the offset of the first element. template struct SmallVectorAlignmentAndSize { - alignas(SmallVectorBase>) char Base[sizeof( - SmallVectorBase>)]; + alignas(small_vector_base>) char Base[sizeof( + small_vector_base>)]; alignas(T) char FirstEl[sizeof(T)]; }; -/// This is the part of SmallVectorTemplateBase which does not depend on whether -/// the type T is a POD. The extra dummy template argument is used by ArrayRef +/// This is the part of small_vector_template_base which does not depend on +/// whether +/// the type T is a POD. The extra dummy template argument is used by array_ref /// to avoid unnecessarily requiring T to be complete. template -class SmallVectorTemplateCommon - : public SmallVectorBase> { - using Base = SmallVectorBase>; +class small_vector_template_common + : public small_vector_base> { + using Base = small_vector_base>; /// Find the address of the first element. For this pointer math to be valid /// with small-size of 0 for T with lots of alignment, it's important that - /// SmallVectorStorage is properly-aligned even for small-size of 0. + /// small_vector_storage is properly-aligned even for small-size of 0. void *getFirstEl() const { return const_cast(reinterpret_cast( reinterpret_cast(this) + @@ -163,7 +165,7 @@ class SmallVectorTemplateCommon // Space after 'FirstEl' is clobbered, do not add any instance vars after it. protected: - SmallVectorTemplateCommon(size_t Size) : Base(getFirstEl(), Size) {} + small_vector_template_common(size_t Size) : Base(getFirstEl(), Size) {} void grow_pod(size_t MinSize, size_t TSize) { Base::grow_pod(getFirstEl(), MinSize, TSize); @@ -358,7 +360,7 @@ class SmallVectorTemplateCommon } }; -/// SmallVectorTemplateBase - This is where we put +/// small_vector_template_base - This is where we put /// method implementations that are designed to work with non-trivial T's. /// /// We approximate is_trivially_copyable with trivial move/copy construction and @@ -370,14 +372,15 @@ template ::value) && (std::is_trivially_move_constructible::value) && std::is_trivially_destructible::value> -class SmallVectorTemplateBase : public SmallVectorTemplateCommon { - friend class SmallVectorTemplateCommon; +class small_vector_template_base : public small_vector_template_common { + friend class small_vector_template_common; protected: static constexpr bool TakesParamByValue = false; using ValueParamT = const T &; - SmallVectorTemplateBase(size_t Size) : SmallVectorTemplateCommon(Size) {} + small_vector_template_base(size_t Size) + : small_vector_template_common(Size) {} static void destroy_range(T *S, T *E) { while (S != E) { @@ -410,7 +413,7 @@ class SmallVectorTemplateBase : public SmallVectorTemplateCommon { /// in \p NewCapacity. This is the first section of \a grow(). T *mallocForGrow(size_t MinSize, size_t &NewCapacity) { return static_cast( - SmallVectorBase>::mallocForGrow( + small_vector_base>::mallocForGrow( MinSize, sizeof(T), NewCapacity)); } @@ -480,7 +483,7 @@ class SmallVectorTemplateBase : public SmallVectorTemplateCommon { // Define this out-of-line to dissuade the C++ compiler from inlining it. template -void SmallVectorTemplateBase::grow(size_t MinSize) { +void small_vector_template_base::grow(size_t MinSize) { size_t NewCapacity; T *NewElts = mallocForGrow(MinSize, NewCapacity); moveElementsForGrow(NewElts); @@ -489,7 +492,7 @@ void SmallVectorTemplateBase::grow(size_t MinSize) { // Define this out-of-line to dissuade the C++ compiler from inlining it. template -void SmallVectorTemplateBase::moveElementsForGrow( +void small_vector_template_base::moveElementsForGrow( T *NewElts) { // Move the elements over. this->uninitialized_move(this->begin(), this->end(), NewElts); @@ -500,7 +503,7 @@ void SmallVectorTemplateBase::moveElementsForGrow( // Define this out-of-line to dissuade the C++ compiler from inlining it. template -void SmallVectorTemplateBase::takeAllocationForGrow( +void small_vector_template_base::takeAllocationForGrow( T *NewElts, size_t NewCapacity) { // If this wasn't grown from the inline copy, deallocate the old space. if (!this->isSmall()) free(this->begin()); @@ -509,13 +512,14 @@ void SmallVectorTemplateBase::takeAllocationForGrow( this->Capacity = NewCapacity; } -/// SmallVectorTemplateBase - This is where we put +/// small_vector_template_base - This is where we put /// method implementations that are designed to work with trivially copyable /// T's. This allows using memcpy in place of copy/move construction and /// skipping destruction. template -class SmallVectorTemplateBase : public SmallVectorTemplateCommon { - friend class SmallVectorTemplateCommon; +class small_vector_template_base + : public small_vector_template_common { + friend class small_vector_template_common; protected: /// True if it's cheap enough to take parameters by value. Doing so avoids @@ -527,7 +531,8 @@ class SmallVectorTemplateBase : public SmallVectorTemplateCommon { using ValueParamT = typename std::conditional::type; - SmallVectorTemplateBase(size_t Size) : SmallVectorTemplateCommon(Size) {} + small_vector_template_base(size_t Size) + : small_vector_template_common(Size) {} // No need to do a destroy loop for POD's. static void destroy_range(T *, T *) {} @@ -557,7 +562,7 @@ class SmallVectorTemplateBase : public SmallVectorTemplateCommon { T2 *Dest, std::enable_if_t::type, T2>::value> * = nullptr) { - // Use memcpy for PODs iterated by pointers (which includes SmallVector + // Use memcpy for PODs iterated by pointers (which includes small_vector // iterators): std::uninitialized_copy optimizes to memmove, but we can // use memcpy here. Note that I and E are iterators and thus might be // invalid for memcpy if they are equal. @@ -612,11 +617,11 @@ class SmallVectorTemplateBase : public SmallVectorTemplateCommon { void pop_back() { this->set_size(this->size() - 1); } }; -/// This class consists of common code factored out of the SmallVector class to -/// reduce code duplication based on the SmallVector 'N' template parameter. +/// This class consists of common code factored out of the small_vector class to +/// reduce code duplication based on the small_vector 'N' template parameter. template -class SmallVectorImpl : public SmallVectorTemplateBase { - using SuperClass = SmallVectorTemplateBase; +class small_vector_impl : public small_vector_template_base { + using SuperClass = small_vector_template_base; public: using iterator = typename SuperClass::iterator; @@ -625,16 +630,16 @@ class SmallVectorImpl : public SmallVectorTemplateBase { using size_type = typename SuperClass::size_type; protected: - using SmallVectorTemplateBase::TakesParamByValue; + using small_vector_template_base::TakesParamByValue; using ValueParamT = typename SuperClass::ValueParamT; // Default ctor - Initialize to empty. - explicit SmallVectorImpl(unsigned N) : SmallVectorTemplateBase(N) {} + explicit small_vector_impl(unsigned N) : small_vector_template_base(N) {} public: - SmallVectorImpl(const SmallVectorImpl &) = delete; + small_vector_impl(const small_vector_impl &) = delete; - ~SmallVectorImpl() { + ~small_vector_impl() { // Subclass has already destructed this vector's elements. // If this wasn't grown from the inline copy, deallocate the old space. if (!this->isSmall()) free(this->begin()); @@ -695,9 +700,9 @@ class SmallVectorImpl : public SmallVectorTemplateBase { return Result; } - void swap(SmallVectorImpl &RHS); + void swap(small_vector_impl &RHS); - /// Add the specified range to the end of the SmallVector. + /// Add the specified range to the end of the small_vector. template ::iterator_category, @@ -719,7 +724,7 @@ class SmallVectorImpl : public SmallVectorTemplateBase { void append(std::initializer_list IL) { append(IL.begin(), IL.end()); } - void append(const SmallVectorImpl &RHS) { append(RHS.begin(), RHS.end()); } + void append(const small_vector_impl &RHS) { append(RHS.begin(), RHS.end()); } void assign(size_type NumElts, ValueParamT Elt) { // Note that Elt could be an internal reference. @@ -755,7 +760,7 @@ class SmallVectorImpl : public SmallVectorTemplateBase { append(IL); } - void assign(const SmallVectorImpl &RHS) { assign(RHS.begin(), RHS.end()); } + void assign(const small_vector_impl &RHS) { assign(RHS.begin(), RHS.end()); } iterator erase(const_iterator CI) { // Just cast away constness because this is a non-const member function. @@ -976,24 +981,26 @@ class SmallVectorImpl : public SmallVectorTemplateBase { return this->back(); } - SmallVectorImpl &operator=(const SmallVectorImpl &RHS); + small_vector_impl &operator=(const small_vector_impl &RHS); - SmallVectorImpl &operator=(SmallVectorImpl &&RHS); + small_vector_impl &operator=(small_vector_impl &&RHS); - bool operator==(const SmallVectorImpl &RHS) const { + bool operator==(const small_vector_impl &RHS) const { if (this->size() != RHS.size()) return false; return std::equal(this->begin(), this->end(), RHS.begin()); } - bool operator!=(const SmallVectorImpl &RHS) const { return !(*this == RHS); } + bool operator!=(const small_vector_impl &RHS) const { + return !(*this == RHS); + } - bool operator<(const SmallVectorImpl &RHS) const { + bool operator<(const small_vector_impl &RHS) const { return std::lexicographical_compare( this->begin(), this->end(), RHS.begin(), RHS.end()); } }; template -void SmallVectorImpl::swap(SmallVectorImpl &RHS) { +void small_vector_impl::swap(small_vector_impl &RHS) { if (this == &RHS) return; // We can only avoid copying elements if neither vector is small. @@ -1028,8 +1035,8 @@ void SmallVectorImpl::swap(SmallVectorImpl &RHS) { } template -SmallVectorImpl &SmallVectorImpl::operator=( - const SmallVectorImpl &RHS) { +small_vector_impl &small_vector_impl::operator=( + const small_vector_impl &RHS) { // Avoid self-assignment. if (this == &RHS) return *this; @@ -1076,7 +1083,8 @@ SmallVectorImpl &SmallVectorImpl::operator=( } template -SmallVectorImpl &SmallVectorImpl::operator=(SmallVectorImpl &&RHS) { +small_vector_impl &small_vector_impl::operator=( + small_vector_impl &&RHS) { // Avoid self-assignment. if (this == &RHS) return *this; @@ -1135,38 +1143,38 @@ SmallVectorImpl &SmallVectorImpl::operator=(SmallVectorImpl &&RHS) { return *this; } -/// Storage for the SmallVector elements. This is specialized for the N=0 case +/// Storage for the small_vector elements. This is specialized for the N=0 case /// to avoid allocating unnecessary storage. template -struct SmallVectorStorage { +struct small_vector_storage { alignas(T) char InlineElts[N * sizeof(T)]; }; /// We need the storage to be properly aligned even for small-size of 0 so that -/// the pointer math in \a SmallVectorTemplateCommon::getFirstEl() is +/// the pointer math in \a small_vector_template_common::getFirstEl() is /// well-defined. template -struct alignas(T) SmallVectorStorage {}; +struct alignas(T) small_vector_storage {}; -/// Forward declaration of SmallVector so that +/// Forward declaration of small_vector so that /// calculateSmallVectorDefaultInlinedElements can reference -/// `sizeof(SmallVector)`. +/// `sizeof(small_vector)`. template -class SmallVector; +class small_vector; /// Helper class for calculating the default number of inline elements for -/// `SmallVector`. +/// `small_vector`. /// /// This should be migrated to a constexpr function when our minimum /// compiler support is enough for multi-statement constexpr functions. template struct CalculateSmallVectorDefaultInlinedElements { // Parameter controlling the default number of inlined elements - // for `SmallVector`. + // for `small_vector`. // // The default number of inlined elements ensures that // 1. There is at least one inlined element. - // 2. `sizeof(SmallVector) <= kPreferredSmallVectorSizeof` unless + // 2. `sizeof(small_vector) <= kPreferredSmallVectorSizeof` unless // it contradicts 1. static constexpr size_t kPreferredSmallVectorSizeof = 64; @@ -1175,14 +1183,14 @@ struct CalculateSmallVectorDefaultInlinedElements { // Because our policy guarantees at least one inlined element, it is possible // for an arbitrarily large inlined element to allocate an arbitrarily large // amount of inline storage. We generally consider it an antipattern for a - // SmallVector to allocate an excessive amount of inline storage, so we want + // small_vector to allocate an excessive amount of inline storage, so we want // to call attention to these cases and make sure that users are making an // intentional decision if they request a lot of inline storage. // // We want this assertion to trigger in pathological cases, but otherwise // not be too easy to hit. To accomplish that, the cutoff is actually somewhat // larger than kPreferredSmallVectorSizeof (otherwise, - // `SmallVector>` would be one easy way to trip it, and that + // `small_vector>` would be one easy way to trip it, and that // pattern seems useful in practice). // // One wrinkle is that this assertion is in theory non-portable, since @@ -1195,14 +1203,14 @@ struct CalculateSmallVectorDefaultInlinedElements { static_assert( sizeof(T) <= 256, "You are trying to use a default number of inlined elements for " - "`SmallVector` but `sizeof(T)` is really big! Please use an " - "explicit number of inlined elements with `SmallVector` to make " + "`small_vector` but `sizeof(T)` is really big! Please use an " + "explicit number of inlined elements with `small_vector` to make " "sure you really want that much inline storage."); // Discount the size of the header itself when calculating the maximum inline // bytes. static constexpr size_t PreferredInlineBytes = - kPreferredSmallVectorSizeof - sizeof(SmallVector); + kPreferredSmallVectorSizeof - sizeof(small_vector); static constexpr size_t NumElementsThatFit = PreferredInlineBytes / sizeof(T); static constexpr size_t value = NumElementsThatFit == 0 ? 1 : NumElementsThatFit; @@ -1216,27 +1224,27 @@ struct CalculateSmallVectorDefaultInlinedElements { /// /// \note /// In the absence of a well-motivated choice for the number of inlined -/// elements \p N, it is recommended to use \c SmallVector (that is, +/// elements \p N, it is recommended to use \c small_vector (that is, /// omitting the \p N). This will choose a default number of inlined elements /// reasonable for allocation on the stack (for example, trying to keep \c -/// sizeof(SmallVector) around 64 bytes). +/// sizeof(small_vector) around 64 bytes). /// /// \warning This does not attempt to be exception safe. /// /// \see https://llvm.org/docs/ProgrammersManual.html#llvm-adt-smallvector-h template ::value> -class SmallVector : public SmallVectorImpl, SmallVectorStorage { +class small_vector : public small_vector_impl, small_vector_storage { public: - SmallVector() : SmallVectorImpl(N) {} + small_vector() : small_vector_impl(N) {} - ~SmallVector() { + ~small_vector() { // Destroy the constructed elements in the vector. this->destroy_range(this->begin(), this->end()); } - explicit SmallVector(size_t Size, const T &Value = T()) - : SmallVectorImpl(N) { + explicit small_vector(size_t Size, const T &Value = T()) + : small_vector_impl(N) { this->assign(Size, Value); } @@ -1244,65 +1252,65 @@ class SmallVector : public SmallVectorImpl, SmallVectorStorage { typename = std::enable_if_t::iterator_category, std::input_iterator_tag>::value>> - SmallVector(ItTy S, ItTy E) : SmallVectorImpl(N) { + small_vector(ItTy S, ItTy E) : small_vector_impl(N) { this->append(S, E); } template - explicit SmallVector(const iterator_range &R) - : SmallVectorImpl(N) { + explicit small_vector(const iterator_range &R) + : small_vector_impl(N) { this->append(R.begin(), R.end()); } - SmallVector(std::initializer_list IL) : SmallVectorImpl(N) { + small_vector(std::initializer_list IL) : small_vector_impl(N) { this->assign(IL); } - SmallVector(const SmallVector &RHS) : SmallVectorImpl(N) { - if (!RHS.empty()) SmallVectorImpl::operator=(RHS); + small_vector(const small_vector &RHS) : small_vector_impl(N) { + if (!RHS.empty()) small_vector_impl::operator=(RHS); } - SmallVector &operator=(const SmallVector &RHS) { - SmallVectorImpl::operator=(RHS); + small_vector &operator=(const small_vector &RHS) { + small_vector_impl::operator=(RHS); return *this; } - SmallVector(SmallVector &&RHS) : SmallVectorImpl(N) { - if (!RHS.empty()) SmallVectorImpl::operator=(::std::move(RHS)); + small_vector(small_vector &&RHS) : small_vector_impl(N) { + if (!RHS.empty()) small_vector_impl::operator=(::std::move(RHS)); } - SmallVector(SmallVectorImpl &&RHS) : SmallVectorImpl(N) { - if (!RHS.empty()) SmallVectorImpl::operator=(::std::move(RHS)); + small_vector(small_vector_impl &&RHS) : small_vector_impl(N) { + if (!RHS.empty()) small_vector_impl::operator=(::std::move(RHS)); } - SmallVector &operator=(SmallVector &&RHS) { - SmallVectorImpl::operator=(::std::move(RHS)); + small_vector &operator=(small_vector &&RHS) { + small_vector_impl::operator=(::std::move(RHS)); return *this; } - SmallVector &operator=(SmallVectorImpl &&RHS) { - SmallVectorImpl::operator=(::std::move(RHS)); + small_vector &operator=(small_vector_impl &&RHS) { + small_vector_impl::operator=(::std::move(RHS)); return *this; } - SmallVector &operator=(std::initializer_list IL) { + small_vector &operator=(std::initializer_list IL) { this->assign(IL); return *this; } }; template -inline size_t capacity_in_bytes(const SmallVector &X) { +inline size_t capacity_in_bytes(const small_vector &X) { return X.capacity_in_bytes(); } /// Given a range of type R, iterate the entire range and return a -/// SmallVector with elements of the vector. This is useful, for example, +/// small_vector with elements of the vector. This is useful, for example, /// when you want to iterate a range and then sort the results. template -SmallVector()))>::type>::type, - Size> +small_vector()))>::type>::type, + Size> to_vector(R &&Range) { return {std::begin(Range), std::end(Range)}; } @@ -1352,22 +1360,22 @@ struct Struct32B { alignas(32) void *X; }; } -static_assert(sizeof(SmallVector) == +static_assert(sizeof(small_vector) == sizeof(unsigned) * 2 + sizeof(void *), - "wasted space in SmallVector size 0"); -static_assert(alignof(SmallVector) >= alignof(Struct16B), + "wasted space in small_vector size 0"); +static_assert(alignof(small_vector) >= alignof(Struct16B), "wrong alignment for 16-byte aligned T"); -static_assert(alignof(SmallVector) >= alignof(Struct32B), +static_assert(alignof(small_vector) >= alignof(Struct32B), "wrong alignment for 32-byte aligned T"); -static_assert(sizeof(SmallVector) >= alignof(Struct16B), +static_assert(sizeof(small_vector) >= alignof(Struct16B), "missing padding for 16-byte aligned T"); -static_assert(sizeof(SmallVector) >= alignof(Struct32B), +static_assert(sizeof(small_vector) >= alignof(Struct32B), "missing padding for 32-byte aligned T"); -static_assert(sizeof(SmallVector) == +static_assert(sizeof(small_vector) == sizeof(unsigned) * 2 + sizeof(void *) * 2, - "wasted space in SmallVector size 1"); + "wasted space in small_vector size 1"); -static_assert(sizeof(SmallVector) == +static_assert(sizeof(small_vector) == sizeof(void *) * 2 + sizeof(void *), "1 byte elements have word-sized type for size and capacity"); @@ -1375,7 +1383,7 @@ static_assert(sizeof(SmallVector) == /// std::length_error or calls report_fatal_error. static void report_size_overflow(size_t MinSize, size_t MaxSize); static void report_size_overflow(size_t MinSize, size_t MaxSize) { - std::string Reason = "SmallVector unable to grow. Requested capacity (" + + std::string Reason = "small_vector unable to grow. Requested capacity (" + std::to_string(MinSize) + ") is larger than maximum value for size type (" + std::to_string(MaxSize) + ")"; @@ -1387,7 +1395,7 @@ static void report_size_overflow(size_t MinSize, size_t MaxSize) { static void report_at_maximum_capacity(size_t MaxSize); static void report_at_maximum_capacity(size_t MaxSize) { std::string Reason = - "SmallVector capacity unable to grow. Already at maximum size " + + "small_vector capacity unable to grow. Already at maximum size " + std::to_string(MaxSize); throw std::length_error(Reason); } @@ -1415,18 +1423,18 @@ static size_t getNewCapacity(size_t MinSize, size_t TSize, size_t OldCapacity) { // Note: Moving this function into the header may cause performance regression. template -void *SmallVectorBase::mallocForGrow(size_t MinSize, - size_t TSize, - size_t &NewCapacity) { +void *small_vector_base::mallocForGrow(size_t MinSize, + size_t TSize, + size_t &NewCapacity) { NewCapacity = getNewCapacity(MinSize, TSize, this->capacity()); return safe_malloc(NewCapacity * TSize); } // Note: Moving this function into the header may cause performance regression. template -void SmallVectorBase::grow_pod(void *FirstEl, - size_t MinSize, - size_t TSize) { +void small_vector_base::grow_pod(void *FirstEl, + size_t MinSize, + size_t TSize) { size_t NewCapacity = getNewCapacity(MinSize, TSize, this->capacity()); void *NewElts; if (BeginX == FirstEl) { @@ -1443,38 +1451,38 @@ void SmallVectorBase::grow_pod(void *FirstEl, this->Capacity = NewCapacity; } -template class paddle::SmallVectorBase; +template class paddle::small_vector_base; // Disable the uint64_t instantiation for 32-bit builds. // Both uint32_t and uint64_t instantiations are needed for 64-bit builds. // This instantiation will never be used in 32-bit builds, and will cause // warnings when sizeof(Size_T) > sizeof(size_t). #if SIZE_MAX > UINT32_MAX -template class paddle::SmallVectorBase; +template class paddle::small_vector_base; // Assertions to ensure this #if stays in sync with SmallVectorSizeType. static_assert(sizeof(SmallVectorSizeType) == sizeof(uint64_t), - "Expected SmallVectorBase variant to be in use."); + "Expected small_vector_base variant to be in use."); #else static_assert(sizeof(SmallVectorSizeType) == sizeof(uint32_t), - "Expected SmallVectorBase variant to be in use."); + "Expected small_vector_base variant to be in use."); #endif } // namespace paddle namespace std { -/// Implement std::swap in terms of SmallVector swap. +/// Implement std::swap in terms of small_vector swap. template -inline void swap(paddle::SmallVectorImpl &LHS, - paddle::SmallVectorImpl &RHS) { +inline void swap(paddle::small_vector_impl &LHS, + paddle::small_vector_impl &RHS) { LHS.swap(RHS); } -/// Implement std::swap in terms of SmallVector swap. +/// Implement std::swap in terms of small_vector swap. template -inline void swap(paddle::SmallVector &LHS, - paddle::SmallVector &RHS) { +inline void swap(paddle::small_vector &LHS, + paddle::small_vector &RHS) { LHS.swap(RHS); } diff --git a/paddle/utils/small_vector_test.cc b/paddle/utils/small_vector_test.cc index 96bcec5940056..e061c232152c5 100644 --- a/paddle/utils/small_vector_test.cc +++ b/paddle/utils/small_vector_test.cc @@ -21,7 +21,7 @@ #include "gtest/gtest.h" template -static std::vector ToStdVector(const paddle::SmallVector &vec) { +static std::vector ToStdVector(const paddle::small_vector &vec) { std::vector std_vec; std_vec.reserve(vec.size()); for (size_t i = 0; i < vec.size(); ++i) { @@ -35,7 +35,7 @@ void SmallVectorCheck(size_t n) { std::srand(std::time(nullptr)); std::vector std_vec; - paddle::SmallVector vec; + paddle::small_vector vec; for (size_t i = 0; i < n; ++i) { int value = rand(); // NOLINT diff --git a/paddle/utils/string/string_helper.h b/paddle/utils/string/string_helper.h index a02b313ef0eba..e6cb2e90b8fa1 100644 --- a/paddle/utils/string/string_helper.h +++ b/paddle/utils/string/string_helper.h @@ -100,6 +100,14 @@ inline int str_to_float(const char* str, float* v) { return index; } +inline float* str_to_float(std::string& str) { + return (float*)const_cast(str.c_str()); +} + +inline float* str_to_float(const char* str) { + return (float*)const_cast(str); +} + // checks whether the test string is a suffix of the input string. bool ends_with(std::string const& input, std::string const& test); diff --git a/paddle/utils/variant.h b/paddle/utils/variant.h new file mode 100644 index 0000000000000..a7546d094c2ff --- /dev/null +++ b/paddle/utils/variant.h @@ -0,0 +1,2830 @@ +// Copy from +// https://github.com/mpark/variant/blob/single-header/v1.4.0/variant.hpp +// Modify the following points: +// 1. modify namespace mpark to namespace paddle + +// MPark.Variant +// +// Copyright Michael Park, 2015-2017 +// +// Distributed under the Boost Software License, Version 1.0. +// (See accompanying file LICENSE.md or copy at +// http://boost.org/LICENSE_1_0.txt) + +#pragma once + +/* + variant synopsis + +namespace std { + + // 20.7.2, class template variant + template + class variant { + public: + + // 20.7.2.1, constructors + constexpr variant() noexcept(see below); + variant(const variant&); + variant(variant&&) noexcept(see below); + + template constexpr variant(T&&) noexcept(see below); + + template + constexpr explicit variant(in_place_type_t, Args&&...); + + template + constexpr explicit variant( + in_place_type_t, initializer_list, Args&&...); + + template + constexpr explicit variant(in_place_index_t, Args&&...); + + template + constexpr explicit variant( + in_place_index_t, initializer_list, Args&&...); + + // 20.7.2.2, destructor + ~variant(); + + // 20.7.2.3, assignment + variant& operator=(const variant&); + variant& operator=(variant&&) noexcept(see below); + + template variant& operator=(T&&) noexcept(see below); + + // 20.7.2.4, modifiers + template + T& emplace(Args&&...); + + template + T& emplace(initializer_list, Args&&...); + + template + variant_alternative& emplace(Args&&...); + + template + variant_alternative& emplace(initializer_list, Args&&...); + + // 20.7.2.5, value status + constexpr bool valueless_by_exception() const noexcept; + constexpr size_t index() const noexcept; + + // 20.7.2.6, swap + void swap(variant&) noexcept(see below); + }; + + // 20.7.3, variant helper classes + template struct variant_size; // undefined + + template + constexpr size_t variant_size_v = variant_size::value; + + template struct variant_size; + template struct variant_size; + template struct variant_size; + + template + struct variant_size>; + + template struct variant_alternative; // undefined + + template + using variant_alternative_t = typename variant_alternative::type; + + template struct variant_alternative; + template struct variant_alternative; + template struct variant_alternative; + + template + struct variant_alternative>; + + constexpr size_t variant_npos = -1; + + // 20.7.4, value access + template + constexpr bool holds_alternative(const variant&) noexcept; + + template + constexpr variant_alternative_t>& + get(variant&); + + template + constexpr variant_alternative_t>&& + get(variant&&); + + template + constexpr variant_alternative_t> const& + get(const variant&); + + template + constexpr variant_alternative_t> const&& + get(const variant&&); + + template + constexpr T& get(variant&); + + template + constexpr T&& get(variant&&); + + template + constexpr const T& get(const variant&); + + template + constexpr const T&& get(const variant&&); + + template + constexpr add_pointer_t>> + get_if(variant*) noexcept; + + template + constexpr add_pointer_t>> + get_if(const variant*) noexcept; + + template + constexpr add_pointer_t + get_if(variant*) noexcept; + + template + constexpr add_pointer_t + get_if(const variant*) noexcept; + + // 20.7.5, relational operators + template + constexpr bool operator==(const variant&, const variant&); + + template + constexpr bool operator!=(const variant&, const variant&); + + template + constexpr bool operator<(const variant&, const variant&); + + template + constexpr bool operator>(const variant&, const variant&); + + template + constexpr bool operator<=(const variant&, const variant&); + + template + constexpr bool operator>=(const variant&, const variant&); + + // 20.7.6, visitation + template + constexpr see below visit(Visitor&&, Variants&&...); + + // 20.7.7, class monostate + struct monostate; + + // 20.7.8, monostate relational operators + constexpr bool operator<(monostate, monostate) noexcept; + constexpr bool operator>(monostate, monostate) noexcept; + constexpr bool operator<=(monostate, monostate) noexcept; + constexpr bool operator>=(monostate, monostate) noexcept; + constexpr bool operator==(monostate, monostate) noexcept; + constexpr bool operator!=(monostate, monostate) noexcept; + + // 20.7.9, specialized algorithms + template + void swap(variant&, variant&) noexcept(see below); + + // 20.7.10, class bad_variant_access + class bad_variant_access; + + // 20.7.11, hash support + template struct hash; + template struct hash>; + template <> struct hash; + +} // namespace std + +*/ + +#pragma once + +#include +#include +#include +#include +#include +#include +#include + +// MPark.Variant +// +// Copyright Michael Park, 2015-2017 +// +// Distributed under the Boost Software License, Version 1.0. +// (See accompanying file LICENSE.md or copy at +// http://boost.org/LICENSE_1_0.txt) + +#ifndef MPARK_CONFIG_HPP +#define MPARK_CONFIG_HPP + +// MSVC 2015 Update 3. +#if __cplusplus < 201103L && (!defined(_MSC_VER) || _MSC_FULL_VER < 190024210) +#error "MPark.Variant requires C++11 support." +#endif + +#ifndef __has_attribute +#define __has_attribute(x) 0 +#endif + +#ifndef __has_builtin +#define __has_builtin(x) 0 +#endif + +#ifndef __has_include +#define __has_include(x) 0 +#endif + +#ifndef __has_feature +#define __has_feature(x) 0 +#endif + +#if __has_attribute(always_inline) || defined(__GNUC__) +#define MPARK_ALWAYS_INLINE __attribute__((__always_inline__)) inline +#elif defined(_MSC_VER) +#define MPARK_ALWAYS_INLINE __forceinline +#else +#define MPARK_ALWAYS_INLINE inline +#endif + +#if __has_builtin(__builtin_addressof) || \ + (defined(__GNUC__) && __GNUC__ >= 7) || defined(_MSC_VER) +#define MPARK_BUILTIN_ADDRESSOF +#endif + +#if __has_builtin(__builtin_unreachable) || defined(__GNUC__) +#define MPARK_BUILTIN_UNREACHABLE __builtin_unreachable() +#elif defined(_MSC_VER) +#define MPARK_BUILTIN_UNREACHABLE __assume(false) +#else +#define MPARK_BUILTIN_UNREACHABLE +#endif + +#if __has_builtin(__type_pack_element) +#define MPARK_TYPE_PACK_ELEMENT +#endif + +#if defined(__cpp_constexpr) && __cpp_constexpr >= 200704 && \ + !(defined(__GNUC__) && __GNUC__ == 4 && __GNUC_MINOR__ == 9) +#define MPARK_CPP11_CONSTEXPR +#endif + +#if defined(__cpp_constexpr) && __cpp_constexpr >= 201304 +#define MPARK_CPP14_CONSTEXPR +#endif + +#if __has_feature(cxx_exceptions) || defined(__cpp_exceptions) || \ + (defined(_MSC_VER) && defined(_CPPUNWIND)) +#define MPARK_EXCEPTIONS +#endif + +#if defined(__cpp_generic_lambdas) || defined(_MSC_VER) +#define MPARK_GENERIC_LAMBDAS +#endif + +#if defined(__cpp_lib_integer_sequence) +#define MPARK_INTEGER_SEQUENCE +#endif + +#if defined(__cpp_return_type_deduction) || defined(_MSC_VER) +#define MPARK_RETURN_TYPE_DEDUCTION +#endif + +#if defined(__cpp_lib_transparent_operators) || defined(_MSC_VER) +#define MPARK_TRANSPARENT_OPERATORS +#endif + +#if defined(__cpp_variable_templates) || defined(_MSC_VER) +#define MPARK_VARIABLE_TEMPLATES +#endif + +#if !defined(__GLIBCXX__) || __has_include() // >= libstdc++-5 +#define MPARK_TRIVIALITY_TYPE_TRAITS +#define MPARK_INCOMPLETE_TYPE_TRAITS +#endif + +#endif // MPARK_CONFIG_HPP + +// MPark.Variant +// +// Copyright Michael Park, 2015-2017 +// +// Distributed under the Boost Software License, Version 1.0. +// (See accompanying file LICENSE.md or copy at +// http://boost.org/LICENSE_1_0.txt) + +#ifndef MPARK_IN_PLACE_HPP +#define MPARK_IN_PLACE_HPP + +#include + +namespace paddle { + +struct in_place_t { + explicit in_place_t() = default; +}; + +template +struct in_place_index_t { + explicit in_place_index_t() = default; +}; + +template +struct in_place_type_t { + explicit in_place_type_t() = default; +}; + +#ifdef MPARK_VARIABLE_TEMPLATES +constexpr in_place_t in_place{}; + +template +constexpr in_place_index_t in_place_index{}; + +template +constexpr in_place_type_t in_place_type{}; +#endif + +} // namespace paddle + +#endif // MPARK_IN_PLACE_HPP + +// MPark.Variant +// +// Copyright Michael Park, 2015-2017 +// +// Distributed under the Boost Software License, Version 1.0. +// (See accompanying file LICENSE.md or copy at +// http://boost.org/LICENSE_1_0.txt) + +#ifndef MPARK_LIB_HPP +#define MPARK_LIB_HPP + +#include +#include +#include +#include + +#define MPARK_RETURN(...) \ + noexcept(noexcept(__VA_ARGS__))->decltype(__VA_ARGS__) { return __VA_ARGS__; } + +namespace paddle { +namespace lib { +template +struct identity { + using type = T; +}; + +inline namespace cpp14 { +template +struct array { + constexpr const T &operator[](std::size_t index) const { return data[index]; } + + T data[N == 0 ? 1 : N]; +}; + +template +using add_pointer_t = typename std::add_pointer::type; + +template +using common_type_t = typename std::common_type::type; + +template +using decay_t = typename std::decay::type; + +template +using enable_if_t = typename std::enable_if::type; + +template +using remove_const_t = typename std::remove_const::type; + +template +using remove_reference_t = typename std::remove_reference::type; + +template +inline constexpr T &&forward(remove_reference_t &t) noexcept { + return static_cast(t); +} + +template +inline constexpr T &&forward(remove_reference_t &&t) noexcept { + static_assert(!std::is_lvalue_reference::value, + "can not forward an rvalue as an lvalue"); + return static_cast(t); +} + +template +inline constexpr remove_reference_t &&move(T &&t) noexcept { + return static_cast &&>(t); +} + +#ifdef MPARK_INTEGER_SEQUENCE +using std::integer_sequence; +using std::index_sequence; +using std::make_index_sequence; +using std::index_sequence_for; +#else +template +struct integer_sequence { + using value_type = T; + static constexpr std::size_t size() noexcept { return sizeof...(Is); } +}; + +template +using index_sequence = integer_sequence; + +template +struct make_index_sequence_concat; + +template +struct make_index_sequence_concat, + index_sequence> + : identity> {}; + +template +struct make_index_sequence_impl; + +template +using make_index_sequence = typename make_index_sequence_impl::type; + +template +struct make_index_sequence_impl + : make_index_sequence_concat, + make_index_sequence> {}; + +template <> +struct make_index_sequence_impl<0> : identity> {}; + +template <> +struct make_index_sequence_impl<1> : identity> {}; + +template +using index_sequence_for = make_index_sequence; +#endif + +// +#ifdef MPARK_TRANSPARENT_OPERATORS +using equal_to = std::equal_to<>; +#else +struct equal_to { + template + inline constexpr auto operator()(Lhs &&lhs, Rhs &&rhs) const + MPARK_RETURN(lib::forward(lhs) == lib::forward(rhs)) +}; +#endif + +#ifdef MPARK_TRANSPARENT_OPERATORS +using not_equal_to = std::not_equal_to<>; +#else +struct not_equal_to { + template + inline constexpr auto operator()(Lhs &&lhs, Rhs &&rhs) const + MPARK_RETURN(lib::forward(lhs) != lib::forward(rhs)) +}; +#endif + +#ifdef MPARK_TRANSPARENT_OPERATORS +using less = std::less<>; +#else +struct less { + template + inline constexpr auto operator()(Lhs &&lhs, Rhs &&rhs) const + MPARK_RETURN(lib::forward(lhs) < lib::forward(rhs)) +}; +#endif + +#ifdef MPARK_TRANSPARENT_OPERATORS +using greater = std::greater<>; +#else +struct greater { + template + inline constexpr auto operator()(Lhs &&lhs, Rhs &&rhs) const + MPARK_RETURN(lib::forward(lhs) > lib::forward(rhs)) +}; +#endif + +#ifdef MPARK_TRANSPARENT_OPERATORS +using less_equal = std::less_equal<>; +#else +struct less_equal { + template + inline constexpr auto operator()(Lhs &&lhs, Rhs &&rhs) const + MPARK_RETURN(lib::forward(lhs) <= lib::forward(rhs)) +}; +#endif + +#ifdef MPARK_TRANSPARENT_OPERATORS +using greater_equal = std::greater_equal<>; +#else +struct greater_equal { + template + inline constexpr auto operator()(Lhs &&lhs, Rhs &&rhs) const + MPARK_RETURN(lib::forward(lhs) >= lib::forward(rhs)) +}; +#endif +} // namespace cpp14 + +inline namespace cpp17 { +// +template +using bool_constant = std::integral_constant; + +template +struct voider : identity {}; + +template +using void_t = typename voider::type; + +namespace detail { +namespace swappable { + +using std::swap; + +template +struct is_swappable { + private: + template (), std::declval()))> + inline static std::true_type test(int); + + template + inline static std::false_type test(...); + + public: + static constexpr bool value = decltype(test(0))::value; +}; + +template +struct is_nothrow_swappable { + static constexpr bool value = + noexcept(swap(std::declval(), std::declval())); +}; + +template +struct is_nothrow_swappable : std::false_type {}; + +} // namespace swappable +} // namespace detail + +using detail::swappable::is_swappable; + +template +using is_nothrow_swappable = + detail::swappable::is_nothrow_swappable::value, T>; + +// +namespace detail { + +template +struct is_reference_wrapper : std::false_type {}; + +template +struct is_reference_wrapper> : std::true_type {}; + +template +struct Invoke; + +template <> +struct Invoke { + template + inline static constexpr auto invoke(R T::*pmf, Arg &&arg, Args &&... args) + MPARK_RETURN((lib::forward(arg).*pmf)(lib::forward(args)...)) +}; + +template <> +struct Invoke { + template + inline static constexpr auto invoke(R T::*pmf, Arg &&arg, Args &&... args) + MPARK_RETURN((lib::forward(arg).get().* + pmf)(lib::forward(args)...)) +}; + +template <> +struct Invoke { + template + inline static constexpr auto invoke(R T::*pmf, Arg &&arg, Args &&... args) + MPARK_RETURN(((*lib::forward(arg)).* + pmf)(lib::forward(args)...)) +}; + +template <> +struct Invoke { + template + inline static constexpr auto invoke(R T::*pmo, Arg &&arg) + MPARK_RETURN(lib::forward(arg).*pmo) +}; + +template <> +struct Invoke { + template + inline static constexpr auto invoke(R T::*pmo, Arg &&arg) + MPARK_RETURN(lib::forward(arg).get().*pmo) +}; + +template <> +struct Invoke { + template + inline static constexpr auto invoke(R T::*pmo, Arg &&arg) + MPARK_RETURN((*lib::forward(arg)).*pmo) +}; + +template +inline constexpr auto invoke(R T::*f, Arg &&arg, Args &&... args) MPARK_RETURN( + Invoke::value, + (std::is_base_of>::value + ? 0 + : is_reference_wrapper>::value ? 1 : 2)>:: + invoke(f, lib::forward(arg), lib::forward(args)...)) + +#ifdef _MSC_VER +#pragma warning(push) +#pragma warning(disable : 4100) +#endif + template + inline constexpr auto invoke(F &&f, Args &&... args) + MPARK_RETURN(lib::forward(f)(lib::forward(args)...)) +#ifdef _MSC_VER +#pragma warning(pop) +#endif +} // namespace detail + +template +inline constexpr auto invoke(F &&f, Args &&... args) + MPARK_RETURN(detail::invoke(lib::forward(f), + lib::forward(args)...)) + + namespace detail { + template + struct invoke_result {}; + + template + struct invoke_result< + void_t(), std::declval()...))>, + F, + Args...> : identity(), + std::declval()...))> {}; + +} // namespace detail + +template +using invoke_result = detail::invoke_result; + +template +using invoke_result_t = typename invoke_result::type; + +namespace detail { + +template +struct is_invocable : std::false_type {}; + +template +struct is_invocable>, F, Args...> + : std::true_type {}; + +template +struct is_invocable_r : std::false_type {}; + +template +struct is_invocable_r>, R, F, Args...> + : std::is_convertible, R> {}; + +} // namespace detail + +template +using is_invocable = detail::is_invocable; + +template +using is_invocable_r = detail::is_invocable_r; + +namespace detail { + +template +struct is_nothrow_invocable { + static constexpr bool value = + noexcept(lib::invoke(std::declval(), std::declval()...)); +}; + +template +struct is_nothrow_invocable : std::false_type {}; + +template +struct is_nothrow_invocable_r { + private: + inline static R impl() { + return lib::invoke(std::declval(), std::declval()...); + } + + public: + static constexpr bool value = noexcept(impl()); +}; + +template +struct is_nothrow_invocable_r : std::false_type {}; + +} // namespace detail + +template +using is_nothrow_invocable = + detail::is_nothrow_invocable::value, F, Args...>; + +template +using is_nothrow_invocable_r = detail:: + is_nothrow_invocable_r::value, R, F, Args...>; + +// +#ifdef MPARK_BUILTIN_ADDRESSOF +template +inline constexpr T *addressof(T &arg) noexcept { + return __builtin_addressof(arg); +} +#else +namespace detail { + +namespace has_addressof_impl { + +struct fail; + +template +inline fail operator&(T &&); + +template +inline static constexpr bool impl() { + return (std::is_class::value || std::is_union::value) && + !std::is_same()), fail>::value; +} + +} // namespace has_addressof_impl + +template +using has_addressof = bool_constant()>; + +template +inline constexpr T *addressof(T &arg, std::true_type) noexcept { + return std::addressof(arg); +} + +template +inline constexpr T *addressof(T &arg, std::false_type) noexcept { + return &arg; +} + +} // namespace detail + +template +inline constexpr T *addressof(T &arg) noexcept { + return detail::addressof(arg, detail::has_addressof{}); +} +#endif + +template +inline constexpr T *addressof(const T &&) = delete; + +} // namespace cpp17 + +template +struct remove_all_extents : identity {}; + +template +struct remove_all_extents> : remove_all_extents {}; + +template +using remove_all_extents_t = typename remove_all_extents::type; + +template +using size_constant = std::integral_constant; + +template +struct indexed_type : size_constant { + using type = T; +}; + +template +using all = std::is_same, + integer_sequence>; + +#ifdef MPARK_TYPE_PACK_ELEMENT +template +using type_pack_element_t = __type_pack_element; +#else +template +struct type_pack_element_impl { + private: + template + struct set; + + template + struct set> : indexed_type... {}; + + template + inline static std::enable_if impl(indexed_type); + + inline static std::enable_if impl(...); + + public: + using type = decltype(impl(set>{})); +}; + +template +using type_pack_element = typename type_pack_element_impl::type; + +template +using type_pack_element_t = typename type_pack_element::type; +#endif + +#ifdef MPARK_TRIVIALITY_TYPE_TRAITS +using std::is_trivially_copy_constructible; +using std::is_trivially_move_constructible; +using std::is_trivially_copy_assignable; +using std::is_trivially_move_assignable; +#else +template +struct is_trivially_copy_constructible + : bool_constant::value &&__has_trivial_copy( + T)> {}; + +template +struct is_trivially_move_constructible : bool_constant<__is_trivial(T)> {}; + +template +struct is_trivially_copy_assignable + : bool_constant::value &&__has_trivial_assign( + T)> {}; + +template +struct is_trivially_move_assignable : bool_constant<__is_trivial(T)> {}; +#endif + +template +struct dependent_type : T {}; + +template +struct push_back; + +template +using push_back_t = typename push_back::type; + +template +struct push_back, J> { + using type = index_sequence; +}; + +} // namespace lib +} // namespace paddle + +#undef MPARK_RETURN + +#endif // MPARK_LIB_HPP + +namespace paddle { + +#ifdef MPARK_RETURN_TYPE_DEDUCTION + +#define AUTO auto +#define AUTO_RETURN(...) \ + { return __VA_ARGS__; } + +#define AUTO_REFREF auto && +#define AUTO_REFREF_RETURN(...) \ + { return __VA_ARGS__; } + +#define DECLTYPE_AUTO decltype(auto) +#define DECLTYPE_AUTO_RETURN(...) \ + { return __VA_ARGS__; } + +#else + +#define AUTO auto +#define AUTO_RETURN(...) \ + ->lib::decay_t { return __VA_ARGS__; } + +#define AUTO_REFREF auto +#define AUTO_REFREF_RETURN(...) \ + ->decltype((__VA_ARGS__)) { \ + static_assert(std::is_reference::value, ""); \ + return __VA_ARGS__; \ + } + +#define DECLTYPE_AUTO auto +#define DECLTYPE_AUTO_RETURN(...) \ + ->decltype(__VA_ARGS__) { return __VA_ARGS__; } + +#endif + +class bad_variant_access : public std::exception { + public: + virtual const char *what() const noexcept override { + return "bad_variant_access"; + } +}; + +[[noreturn]] inline void throw_bad_variant_access() { +#ifdef MPARK_EXCEPTIONS + throw bad_variant_access{}; +#else + std::terminate(); + MPARK_BUILTIN_UNREACHABLE; +#endif +} + +template +class variant; + +template +struct variant_size; + +#ifdef MPARK_VARIABLE_TEMPLATES +template +constexpr std::size_t variant_size_v = variant_size::value; +#endif + +template +struct variant_size : variant_size {}; + +template +struct variant_size : variant_size {}; + +template +struct variant_size : variant_size {}; + +template +struct variant_size> : lib::size_constant {}; + +template +struct variant_alternative; + +template +using variant_alternative_t = typename variant_alternative::type; + +template +struct variant_alternative + : std::add_const> {}; + +template +struct variant_alternative + : std::add_volatile> {}; + +template +struct variant_alternative + : std::add_cv> {}; + +template +struct variant_alternative> { + static_assert(I < sizeof...(Ts), + "index out of bounds in `std::variant_alternative<>`"); + using type = lib::type_pack_element_t; +}; + +constexpr std::size_t variant_npos = static_cast(-1); + +namespace detail { + +constexpr std::size_t not_found = static_cast(-1); +constexpr std::size_t ambiguous = static_cast(-2); + +#ifdef MPARK_CPP14_CONSTEXPR +template +inline constexpr std::size_t find_index() { + constexpr lib::array matches = { + {std::is_same::value...}}; + std::size_t result = not_found; + for (std::size_t i = 0; i < sizeof...(Ts); ++i) { + if (matches[i]) { + if (result != not_found) { + return ambiguous; + } + result = i; + } + } + return result; +} +#else +inline constexpr std::size_t find_index_impl(std::size_t result, std::size_t) { + return result; +} + +template +inline constexpr std::size_t find_index_impl(std::size_t result, + std::size_t idx, + bool b, + Bs... bs) { + return b ? (result != not_found ? ambiguous + : find_index_impl(idx, idx + 1, bs...)) + : find_index_impl(result, idx + 1, bs...); +} + +template +inline constexpr std::size_t find_index() { + return find_index_impl(not_found, 0, std::is_same::value...); +} +#endif + +template +using find_index_sfinae_impl = + lib::enable_if_t>; + +template +using find_index_sfinae = find_index_sfinae_impl()>; + +template +struct find_index_checked_impl : lib::size_constant { + static_assert(I != not_found, "the specified type is not found."); + static_assert(I != ambiguous, "the specified type is ambiguous."); +}; + +template +using find_index_checked = find_index_checked_impl()>; + +struct valueless_t {}; + +enum class Trait { TriviallyAvailable, Available, Unavailable }; + +template class IsTriviallyAvailable, + template class IsAvailable> +inline constexpr Trait trait() { + return IsTriviallyAvailable::value + ? Trait::TriviallyAvailable + : IsAvailable::value ? Trait::Available : Trait::Unavailable; +} + +#ifdef MPARK_CPP14_CONSTEXPR +template +inline constexpr Trait common_trait(Traits... traits_) { + Trait result = Trait::TriviallyAvailable; + lib::array traits = {{traits_...}}; + for (std::size_t i = 0; i < sizeof...(Traits); ++i) { + Trait t = traits[i]; + if (static_cast(t) > static_cast(result)) { + result = t; + } + } + return result; +} +#else +inline constexpr Trait common_trait_impl(Trait result) { return result; } + +template +inline constexpr Trait common_trait_impl(Trait result, Trait t, Traits... ts) { + return static_cast(t) > static_cast(result) + ? common_trait_impl(t, ts...) + : common_trait_impl(result, ts...); +} + +template +inline constexpr Trait common_trait(Traits... ts) { + return common_trait_impl(Trait::TriviallyAvailable, ts...); +} +#endif + +template +struct traits { + static constexpr Trait copy_constructible_trait = + common_trait(trait()...); + + static constexpr Trait move_constructible_trait = + common_trait(trait()...); + + static constexpr Trait copy_assignable_trait = + common_trait(copy_constructible_trait, + trait()...); + + static constexpr Trait move_assignable_trait = + common_trait(move_constructible_trait, + trait()...); + + static constexpr Trait destructible_trait = common_trait( + trait()...); +}; + +namespace access { + +struct recursive_union { +#ifdef MPARK_RETURN_TYPE_DEDUCTION + template + inline static constexpr auto &&get_alt(V &&v, in_place_index_t<0>) { + return lib::forward(v).head_; + } + + template + inline static constexpr auto &&get_alt(V &&v, in_place_index_t) { + return get_alt(lib::forward(v).tail_, in_place_index_t{}); + } +#else + template + struct get_alt_impl { + template + inline constexpr AUTO_REFREF operator()(V &&v) const + AUTO_REFREF_RETURN(get_alt_impl{}(lib::forward(v).tail_)) + }; + + template + struct get_alt_impl<0, Dummy> { + template + inline constexpr AUTO_REFREF operator()(V &&v) const + AUTO_REFREF_RETURN(lib::forward(v).head_) + }; + + template + inline static constexpr AUTO_REFREF get_alt(V &&v, in_place_index_t) + AUTO_REFREF_RETURN(get_alt_impl{}(lib::forward(v))) +#endif +}; + +struct base { + template + inline static constexpr AUTO_REFREF get_alt(V &&v) +#ifdef _MSC_VER + AUTO_REFREF_RETURN(recursive_union::get_alt(lib::forward(v).data_, + in_place_index_t{})) +#else + AUTO_REFREF_RETURN(recursive_union::get_alt(data(lib::forward(v)), + in_place_index_t{})) +#endif +}; + +struct variant { + template + inline static constexpr AUTO_REFREF get_alt(V &&v) + AUTO_REFREF_RETURN(base::get_alt(lib::forward(v).impl_)) +}; + +} // namespace access + +namespace visitation { + +#if defined(MPARK_CPP14_CONSTEXPR) && !defined(_MSC_VER) +#define MPARK_VARIANT_SWITCH_VISIT +#endif + +struct base { + template + using dispatch_result_t = + decltype(lib::invoke(std::declval(), + access::base::get_alt<0>(std::declval())...)); + + template + struct expected { + template + inline static constexpr bool but_got() { + return std::is_same::value; + } + }; + + template + struct visit_return_type_check { + static_assert(expected::template but_got(), + "`visit` requires the visitor to have a single return type"); + + template + inline static constexpr DECLTYPE_AUTO invoke(Visitor &&visitor, + Alts &&... alts) + DECLTYPE_AUTO_RETURN(lib::invoke(lib::forward(visitor), + lib::forward(alts)...)) + }; + +#ifdef MPARK_VARIANT_SWITCH_VISIT + template + struct dispatcher; + + template + struct dispatcher { + template + MPARK_ALWAYS_INLINE static constexpr R dispatch(F &&, + typename ITs::type &&..., + Vs &&...) { + MPARK_BUILTIN_UNREACHABLE; + } + + template + MPARK_ALWAYS_INLINE static constexpr R dispatch_case(F &&, Vs &&...) { + MPARK_BUILTIN_UNREACHABLE; + } + + template + MPARK_ALWAYS_INLINE static constexpr R dispatch_at(std::size_t, + F &&, + Vs &&...) { + MPARK_BUILTIN_UNREACHABLE; + } + }; + + template + struct dispatcher { + template + MPARK_ALWAYS_INLINE static constexpr R dispatch( + F &&f, typename ITs::type &&... visited_vs) { + using Expected = R; + using Actual = decltype( + lib::invoke(lib::forward(f), + access::base::get_alt( + lib::forward(visited_vs))...)); + return visit_return_type_check::invoke( + lib::forward(f), + access::base::get_alt( + lib::forward(visited_vs))...); + } + + template + MPARK_ALWAYS_INLINE static constexpr R dispatch( + F &&f, typename ITs::type &&... visited_vs, V &&v, Vs &&... vs) { +#define MPARK_DISPATCH(I) \ + dispatcher<(I < lib::decay_t::size()), \ + R, \ + ITs..., \ + lib::indexed_type>:: \ + template dispatch<0>(lib::forward(f), \ + lib::forward(visited_vs)..., \ + lib::forward(v), \ + lib::forward(vs)...) + +#define MPARK_DEFAULT(I) \ + dispatcher<(I < lib::decay_t::size()), R, ITs...>::template dispatch( \ + lib::forward(f), \ + lib::forward(visited_vs)..., \ + lib::forward(v), \ + lib::forward(vs)...) + + switch (v.index()) { + case B + 0: + return MPARK_DISPATCH(B + 0); + case B + 1: + return MPARK_DISPATCH(B + 1); + case B + 2: + return MPARK_DISPATCH(B + 2); + case B + 3: + return MPARK_DISPATCH(B + 3); + case B + 4: + return MPARK_DISPATCH(B + 4); + case B + 5: + return MPARK_DISPATCH(B + 5); + case B + 6: + return MPARK_DISPATCH(B + 6); + case B + 7: + return MPARK_DISPATCH(B + 7); + case B + 8: + return MPARK_DISPATCH(B + 8); + case B + 9: + return MPARK_DISPATCH(B + 9); + case B + 10: + return MPARK_DISPATCH(B + 10); + case B + 11: + return MPARK_DISPATCH(B + 11); + case B + 12: + return MPARK_DISPATCH(B + 12); + case B + 13: + return MPARK_DISPATCH(B + 13); + case B + 14: + return MPARK_DISPATCH(B + 14); + case B + 15: + return MPARK_DISPATCH(B + 15); + case B + 16: + return MPARK_DISPATCH(B + 16); + case B + 17: + return MPARK_DISPATCH(B + 17); + case B + 18: + return MPARK_DISPATCH(B + 18); + case B + 19: + return MPARK_DISPATCH(B + 19); + case B + 20: + return MPARK_DISPATCH(B + 20); + case B + 21: + return MPARK_DISPATCH(B + 21); + case B + 22: + return MPARK_DISPATCH(B + 22); + case B + 23: + return MPARK_DISPATCH(B + 23); + case B + 24: + return MPARK_DISPATCH(B + 24); + case B + 25: + return MPARK_DISPATCH(B + 25); + case B + 26: + return MPARK_DISPATCH(B + 26); + case B + 27: + return MPARK_DISPATCH(B + 27); + case B + 28: + return MPARK_DISPATCH(B + 28); + case B + 29: + return MPARK_DISPATCH(B + 29); + case B + 30: + return MPARK_DISPATCH(B + 30); + case B + 31: + return MPARK_DISPATCH(B + 31); + default: + return MPARK_DEFAULT(B + 32); + } + +#undef MPARK_DEFAULT +#undef MPARK_DISPATCH + } + + template + MPARK_ALWAYS_INLINE static constexpr R dispatch_case(F &&f, Vs &&... vs) { + using Expected = R; + using Actual = decltype( + lib::invoke(lib::forward(f), + access::base::get_alt(lib::forward(vs))...)); + return visit_return_type_check::invoke( + lib::forward(f), + access::base::get_alt(lib::forward(vs))...); + } + + template + MPARK_ALWAYS_INLINE static constexpr R dispatch_at(std::size_t index, + F &&f, + V &&v, + Vs &&... vs) { + static_assert(lib::all<(lib::decay_t::size() == + lib::decay_t::size())...>::value, + "all of the variants must be the same size."); +#define MPARK_DISPATCH_AT(I) \ + dispatcher<(I < lib::decay_t::size()), R>::template dispatch_case( \ + lib::forward(f), lib::forward(v), lib::forward(vs)...) + +#define MPARK_DEFAULT(I) \ + dispatcher<(I < lib::decay_t::size()), R>::template dispatch_at( \ + index, lib::forward(f), lib::forward(v), lib::forward(vs)...) + + switch (index) { + case B + 0: + return MPARK_DISPATCH_AT(B + 0); + case B + 1: + return MPARK_DISPATCH_AT(B + 1); + case B + 2: + return MPARK_DISPATCH_AT(B + 2); + case B + 3: + return MPARK_DISPATCH_AT(B + 3); + case B + 4: + return MPARK_DISPATCH_AT(B + 4); + case B + 5: + return MPARK_DISPATCH_AT(B + 5); + case B + 6: + return MPARK_DISPATCH_AT(B + 6); + case B + 7: + return MPARK_DISPATCH_AT(B + 7); + case B + 8: + return MPARK_DISPATCH_AT(B + 8); + case B + 9: + return MPARK_DISPATCH_AT(B + 9); + case B + 10: + return MPARK_DISPATCH_AT(B + 10); + case B + 11: + return MPARK_DISPATCH_AT(B + 11); + case B + 12: + return MPARK_DISPATCH_AT(B + 12); + case B + 13: + return MPARK_DISPATCH_AT(B + 13); + case B + 14: + return MPARK_DISPATCH_AT(B + 14); + case B + 15: + return MPARK_DISPATCH_AT(B + 15); + case B + 16: + return MPARK_DISPATCH_AT(B + 16); + case B + 17: + return MPARK_DISPATCH_AT(B + 17); + case B + 18: + return MPARK_DISPATCH_AT(B + 18); + case B + 19: + return MPARK_DISPATCH_AT(B + 19); + case B + 20: + return MPARK_DISPATCH_AT(B + 20); + case B + 21: + return MPARK_DISPATCH_AT(B + 21); + case B + 22: + return MPARK_DISPATCH_AT(B + 22); + case B + 23: + return MPARK_DISPATCH_AT(B + 23); + case B + 24: + return MPARK_DISPATCH_AT(B + 24); + case B + 25: + return MPARK_DISPATCH_AT(B + 25); + case B + 26: + return MPARK_DISPATCH_AT(B + 26); + case B + 27: + return MPARK_DISPATCH_AT(B + 27); + case B + 28: + return MPARK_DISPATCH_AT(B + 28); + case B + 29: + return MPARK_DISPATCH_AT(B + 29); + case B + 30: + return MPARK_DISPATCH_AT(B + 30); + case B + 31: + return MPARK_DISPATCH_AT(B + 31); + default: + return MPARK_DEFAULT(B + 32); + } + +#undef MPARK_DEFAULT +#undef MPARK_DISPATCH_AT + } + }; +#else + template + inline static constexpr const T &at(const T &elem) noexcept { + return elem; + } + + template + inline static constexpr const lib::remove_all_extents_t &at( + const lib::array &elems, std::size_t i, Is... is) noexcept { + return at(elems[i], is...); + } + + template + inline static constexpr lib::array, sizeof...(Fs) + 1> + make_farray(F &&f, Fs &&... fs) { + return {{lib::forward(f), lib::forward(fs)...}}; + } + + template + struct make_fmatrix_impl { + template + inline static constexpr dispatch_result_t dispatch(F &&f, + Vs &&... vs) { + using Expected = dispatch_result_t; + using Actual = decltype( + lib::invoke(lib::forward(f), + access::base::get_alt(lib::forward(vs))...)); + return visit_return_type_check::invoke( + lib::forward(f), + access::base::get_alt(lib::forward(vs))...); + } + +#ifdef MPARK_RETURN_TYPE_DEDUCTION + template + inline static constexpr auto impl(lib::index_sequence) { + return &dispatch; + } + + template + inline static constexpr auto impl(Is, + lib::index_sequence, + Ls... ls) { + return make_farray(impl(lib::push_back_t{}, ls...)...); + } +#else + template + struct impl; + + template + struct impl> { + inline constexpr AUTO operator()() const AUTO_RETURN(&dispatch) + }; + + template + struct impl, Ls...> { + inline constexpr AUTO operator()() const + AUTO_RETURN(make_farray(impl, Ls...>{}()...)) + }; +#endif + }; + +#ifdef MPARK_RETURN_TYPE_DEDUCTION + template + inline static constexpr auto make_fmatrix() { + return make_fmatrix_impl::impl( + lib::index_sequence<>{}, + lib::make_index_sequence::size()>{}...); + } +#else + template + inline static constexpr AUTO make_fmatrix() + AUTO_RETURN(typename make_fmatrix_impl::template impl< + lib::index_sequence<>, + lib::make_index_sequence::size()>...>{}()) +#endif + + template + struct make_fdiagonal_impl { + template + inline static constexpr dispatch_result_t dispatch(F &&f, + Vs &&... vs) { + using Expected = dispatch_result_t; + using Actual = decltype( + lib::invoke(lib::forward(f), + access::base::get_alt(lib::forward(vs))...)); + return visit_return_type_check::invoke( + lib::forward(f), + access::base::get_alt(lib::forward(vs))...); + } + + template + inline static constexpr AUTO impl(lib::index_sequence) + AUTO_RETURN(make_farray(&dispatch...)) + }; + + template + inline static constexpr auto make_fdiagonal() + -> decltype(make_fdiagonal_impl::impl( + lib::make_index_sequence::size()>{})) { + static_assert(lib::all<(lib::decay_t::size() == + lib::decay_t::size())...>::value, + "all of the variants must be the same size."); + return make_fdiagonal_impl::impl( + lib::make_index_sequence::size()>{}); + } +#endif +}; + +#if !defined(MPARK_VARIANT_SWITCH_VISIT) && \ + (!defined(_MSC_VER) || _MSC_VER >= 1910) +template +using fmatrix_t = decltype(base::make_fmatrix()); + +template +struct fmatrix { + static constexpr fmatrix_t value = base::make_fmatrix(); +}; + +template +constexpr fmatrix_t fmatrix::value; + +template +using fdiagonal_t = decltype(base::make_fdiagonal()); + +template +struct fdiagonal { + static constexpr fdiagonal_t value = + base::make_fdiagonal(); +}; + +template +constexpr fdiagonal_t fdiagonal::value; +#endif + +struct alt { + template + inline static constexpr DECLTYPE_AUTO visit_alt(Visitor &&visitor, + Vs &&... vs) +#ifdef MPARK_VARIANT_SWITCH_VISIT + DECLTYPE_AUTO_RETURN( + base::dispatcher(vs)))...>>:: + template dispatch<0>(lib::forward(visitor), + as_base(lib::forward(vs))...)) +#elif !defined(_MSC_VER) || _MSC_VER >= 1910 + DECLTYPE_AUTO_RETURN( + base::at(fmatrix(vs)))...>::value, + vs.index()...)(lib::forward(visitor), + as_base(lib::forward(vs))...)) +#else + DECLTYPE_AUTO_RETURN(base::at( + base::make_fmatrix(vs)))...>(), + vs.index()...)(lib::forward(visitor), + as_base(lib::forward(vs))...)) +#endif + + template + inline static constexpr DECLTYPE_AUTO + visit_alt_at(std::size_t index, Visitor &&visitor, Vs &&... vs) +#ifdef MPARK_VARIANT_SWITCH_VISIT + DECLTYPE_AUTO_RETURN( + base::dispatcher< + true, + base::dispatch_result_t< + Visitor, + decltype(as_base(lib::forward(vs)))...>>:: + template dispatch_at<0>(index, + lib::forward(visitor), + as_base(lib::forward(vs))...)) +#elif !defined(_MSC_VER) || _MSC_VER >= 1910 + DECLTYPE_AUTO_RETURN(base::at( + fdiagonal(vs)))...>::value, + index)(lib::forward(visitor), + as_base(lib::forward(vs))...)) +#else + DECLTYPE_AUTO_RETURN( + base::at(base::make_fdiagonal< + Visitor &&, + decltype(as_base(lib::forward(vs)))...>(), + index)(lib::forward(visitor), + as_base(lib::forward(vs))...)) +#endif +}; + +struct variant { + private: + template + struct visitor { + template + inline static constexpr bool does_not_handle() { + return lib::is_invocable::value; + } + }; + + template + struct visit_exhaustiveness_check { + static_assert(visitor::template does_not_handle(), + "`visit` requires the visitor to be exhaustive."); + + inline static constexpr DECLTYPE_AUTO invoke(Visitor &&visitor, + Values &&... values) + DECLTYPE_AUTO_RETURN(lib::invoke(lib::forward(visitor), + lib::forward(values)...)) + }; + + template + struct value_visitor { + Visitor &&visitor_; + + template + inline constexpr DECLTYPE_AUTO operator()(Alts &&... alts) const + DECLTYPE_AUTO_RETURN(visit_exhaustiveness_check< + Visitor, + decltype((lib::forward(alts).value))...>:: + invoke(lib::forward(visitor_), + lib::forward(alts).value...)) + }; + + template + inline static constexpr AUTO make_value_visitor(Visitor &&visitor) + AUTO_RETURN(value_visitor{lib::forward(visitor)}) + + public + : template + inline static constexpr DECLTYPE_AUTO + visit_alt(Visitor &&visitor, Vs &&... vs) + DECLTYPE_AUTO_RETURN(alt::visit_alt(lib::forward(visitor), + lib::forward(vs).impl_...)) + + template + inline static constexpr DECLTYPE_AUTO + visit_alt_at(std::size_t index, Visitor &&visitor, Vs &&... vs) + DECLTYPE_AUTO_RETURN( + alt::visit_alt_at(index, + lib::forward(visitor), + lib::forward(vs).impl_...)) + + template + inline static constexpr DECLTYPE_AUTO + visit_value(Visitor &&visitor, Vs &&... vs) DECLTYPE_AUTO_RETURN( + visit_alt(make_value_visitor(lib::forward(visitor)), + lib::forward(vs)...)) + + template + inline static constexpr DECLTYPE_AUTO + visit_value_at(std::size_t index, Visitor &&visitor, Vs &&... vs) + DECLTYPE_AUTO_RETURN( + visit_alt_at(index, + make_value_visitor(lib::forward(visitor)), + lib::forward(vs)...)) +}; + +} // namespace visitation + +template +struct alt { + using value_type = T; + +#ifdef _MSC_VER +#pragma warning(push) +#pragma warning(disable : 4244) +#endif + template + inline explicit constexpr alt(in_place_t, Args &&... args) + : value(lib::forward(args)...) {} +#ifdef _MSC_VER +#pragma warning(pop) +#endif + + T value; +}; + +template +union recursive_union; + +template +union recursive_union {}; + +#define MPARK_VARIANT_RECURSIVE_UNION(destructible_trait, destructor) \ + template \ + union recursive_union { \ + public: \ + inline explicit constexpr recursive_union(valueless_t) noexcept \ + : dummy_{} {} \ + \ + template \ + inline explicit constexpr recursive_union(in_place_index_t<0>, \ + Args &&... args) \ + : head_(in_place_t{}, lib::forward(args)...) {} \ + \ + template \ + inline explicit constexpr recursive_union(in_place_index_t, \ + Args &&... args) \ + : tail_(in_place_index_t{}, lib::forward(args)...) {} \ + \ + recursive_union(const recursive_union &) = default; \ + recursive_union(recursive_union &&) = default; \ + \ + destructor \ + \ + recursive_union & \ + operator=(const recursive_union &) = default; \ + recursive_union &operator=(recursive_union &&) = default; \ + \ + private: \ + char dummy_; \ + alt head_; \ + recursive_union tail_; \ + \ + friend struct access::recursive_union; \ + } + +MPARK_VARIANT_RECURSIVE_UNION(Trait::TriviallyAvailable, + ~recursive_union() = default;); +MPARK_VARIANT_RECURSIVE_UNION(Trait::Available, ~recursive_union(){}); +MPARK_VARIANT_RECURSIVE_UNION(Trait::Unavailable, ~recursive_union() = delete;); + +#undef MPARK_VARIANT_RECURSIVE_UNION + +using index_t = unsigned int; + +template +class base { + public: + inline explicit constexpr base(valueless_t tag) noexcept + : data_(tag), + index_(static_cast(-1)) {} + + template + inline explicit constexpr base(in_place_index_t, Args &&... args) + : data_(in_place_index_t{}, lib::forward(args)...), index_(I) {} + + inline constexpr bool valueless_by_exception() const noexcept { + return index_ == static_cast(-1); + } + + inline constexpr std::size_t index() const noexcept { + return valueless_by_exception() ? variant_npos : index_; + } + + protected: + using data_t = recursive_union; + + friend inline constexpr base &as_base(base &b) { return b; } + friend inline constexpr const base &as_base(const base &b) { return b; } + friend inline constexpr base &&as_base(base &&b) { return lib::move(b); } + friend inline constexpr const base &&as_base(const base &&b) { + return lib::move(b); + } + + friend inline constexpr data_t &data(base &b) { return b.data_; } + friend inline constexpr const data_t &data(const base &b) { return b.data_; } + friend inline constexpr data_t &&data(base &&b) { return lib::move(b).data_; } + friend inline constexpr const data_t &&data(const base &&b) { + return lib::move(b).data_; + } + + inline static constexpr std::size_t size() { return sizeof...(Ts); } + + data_t data_; + index_t index_; + + friend struct access::base; + friend struct visitation::base; +}; + +struct dtor { +#ifdef _MSC_VER +#pragma warning(push) +#pragma warning(disable : 4100) +#endif + template + inline void operator()(Alt &alt) const noexcept { + alt.~Alt(); + } +#ifdef _MSC_VER +#pragma warning(pop) +#endif +}; + +#if !defined(_MSC_VER) || _MSC_VER >= 1910 +#define MPARK_INHERITING_CTOR(type, base) using base::base; +#else +#define MPARK_INHERITING_CTOR(type, base) \ + template \ + inline explicit constexpr type(Args &&... args) \ + : base(lib::forward(args)...) {} +#endif + +template +class destructor; + +#define MPARK_VARIANT_DESTRUCTOR(destructible_trait, definition, destroy) \ + template \ + class destructor, destructible_trait> \ + : public base { \ + using super = base; \ + \ + public: \ + MPARK_INHERITING_CTOR(destructor, super) \ + using super::operator=; \ + \ + destructor(const destructor &) = default; \ + destructor(destructor &&) = default; \ + definition destructor &operator=(const destructor &) = default; \ + destructor &operator=(destructor &&) = default; \ + \ + protected: \ + destroy \ + } + +MPARK_VARIANT_DESTRUCTOR(Trait::TriviallyAvailable, ~destructor() = default; + , inline void destroy() noexcept { + this->index_ = static_cast(-1); + }); + +MPARK_VARIANT_DESTRUCTOR(Trait::Available, + ~destructor() { destroy(); }, + inline void destroy() noexcept { + if (!this->valueless_by_exception()) { + visitation::alt::visit_alt(dtor{}, *this); + } + this->index_ = static_cast(-1); + }); + +MPARK_VARIANT_DESTRUCTOR(Trait::Unavailable, ~destructor() = delete; + , inline void destroy() noexcept = delete;); + +#undef MPARK_VARIANT_DESTRUCTOR + +template +class constructor : public destructor { + using super = destructor; + + public: + MPARK_INHERITING_CTOR(constructor, super) + using super::operator=; + + protected: +#ifndef MPARK_GENERIC_LAMBDAS + struct ctor { + template + inline void operator()(LhsAlt &lhs_alt, RhsAlt &&rhs_alt) const { + constructor::construct_alt(lhs_alt, lib::forward(rhs_alt).value); + } + }; +#endif + + template + inline static T &construct_alt(alt &a, Args &&... args) { + auto *result = ::new (static_cast(lib::addressof(a))) + alt(in_place_t{}, lib::forward(args)...); + return result->value; + } + + template + inline static void generic_construct(constructor &lhs, Rhs &&rhs) { + lhs.destroy(); + if (!rhs.valueless_by_exception()) { + visitation::alt::visit_alt_at( + rhs.index(), +#ifdef MPARK_GENERIC_LAMBDAS + [](auto &lhs_alt, auto &&rhs_alt) { + constructor::construct_alt( + lhs_alt, lib::forward(rhs_alt).value); + } +#else + ctor {} +#endif + , + lhs, + lib::forward(rhs)); + lhs.index_ = rhs.index_; + } + } +}; + +template +class move_constructor; + +#define MPARK_VARIANT_MOVE_CONSTRUCTOR(move_constructible_trait, definition) \ + template \ + class move_constructor, move_constructible_trait> \ + : public constructor> { \ + using super = constructor>; \ + \ + public: \ + MPARK_INHERITING_CTOR(move_constructor, super) \ + using super::operator=; \ + \ + move_constructor(const move_constructor &) = default; \ + definition ~move_constructor() = default; \ + move_constructor &operator=(const move_constructor &) = default; \ + move_constructor &operator=(move_constructor &&) = default; \ + } + +MPARK_VARIANT_MOVE_CONSTRUCTOR( + Trait::TriviallyAvailable, + move_constructor(move_constructor &&that) = default;); + +MPARK_VARIANT_MOVE_CONSTRUCTOR( + Trait::Available, + move_constructor(move_constructor &&that) noexcept( + lib::all::value...>::value) + : move_constructor(valueless_t{}) { + this->generic_construct(*this, lib::move(that)); + }); + +MPARK_VARIANT_MOVE_CONSTRUCTOR(Trait::Unavailable, + move_constructor(move_constructor &&) = delete;); + +#undef MPARK_VARIANT_MOVE_CONSTRUCTOR + +template +class copy_constructor; + +#define MPARK_VARIANT_COPY_CONSTRUCTOR(copy_constructible_trait, definition) \ + template \ + class copy_constructor, copy_constructible_trait> \ + : public move_constructor> { \ + using super = move_constructor>; \ + \ + public: \ + MPARK_INHERITING_CTOR(copy_constructor, super) \ + using super::operator=; \ + \ + definition copy_constructor(copy_constructor &&) = default; \ + ~copy_constructor() = default; \ + copy_constructor &operator=(const copy_constructor &) = default; \ + copy_constructor &operator=(copy_constructor &&) = default; \ + } + +MPARK_VARIANT_COPY_CONSTRUCTOR( + Trait::TriviallyAvailable, + copy_constructor(const copy_constructor &that) = default;); + +MPARK_VARIANT_COPY_CONSTRUCTOR(Trait::Available, + copy_constructor(const copy_constructor &that) + : copy_constructor(valueless_t{}) { + this->generic_construct(*this, that); + }); + +MPARK_VARIANT_COPY_CONSTRUCTOR( + Trait::Unavailable, copy_constructor(const copy_constructor &) = delete;); + +#undef MPARK_VARIANT_COPY_CONSTRUCTOR + +template +class assignment : public copy_constructor { + using super = copy_constructor; + + public: + MPARK_INHERITING_CTOR(assignment, super) + using super::operator=; + + template + inline /* auto & */ auto emplace(Args &&... args) + -> decltype(this->construct_alt(access::base::get_alt(*this), + lib::forward(args)...)) { + this->destroy(); + auto &result = this->construct_alt(access::base::get_alt(*this), + lib::forward(args)...); + this->index_ = I; + return result; + } + + protected: +#ifndef MPARK_GENERIC_LAMBDAS + template + struct assigner { + template + inline void operator()(ThisAlt &this_alt, ThatAlt &&that_alt) const { + self->assign_alt(this_alt, lib::forward(that_alt).value); + } + assignment *self; + }; +#endif + + template + inline void assign_alt(alt &a, Arg &&arg) { + if (this->index() == I) { +#ifdef _MSC_VER +#pragma warning(push) +#pragma warning(disable : 4244) +#endif + a.value = lib::forward(arg); +#ifdef _MSC_VER +#pragma warning(pop) +#endif + } else { + struct { + void operator()(std::true_type) const { + this_->emplace(lib::forward(arg_)); + } + void operator()(std::false_type) const { + this_->emplace(T(lib::forward(arg_))); + } + assignment *this_; + Arg &&arg_; + } impl{this, lib::forward(arg)}; + impl(lib::bool_constant < std::is_nothrow_constructible::value || + !std::is_nothrow_move_constructible::value > {}); + } + } + + template + inline void generic_assign(That &&that) { + if (this->valueless_by_exception() && that.valueless_by_exception()) { + // do nothing. + } else if (that.valueless_by_exception()) { + this->destroy(); + } else { + visitation::alt::visit_alt_at( + that.index(), +#ifdef MPARK_GENERIC_LAMBDAS + [this](auto &this_alt, auto &&that_alt) { + this->assign_alt(this_alt, + lib::forward(that_alt).value); + } +#else + assigner { this } +#endif + , + *this, + lib::forward(that)); + } + } +}; + +template +class move_assignment; + +#define MPARK_VARIANT_MOVE_ASSIGNMENT(move_assignable_trait, definition) \ + template \ + class move_assignment, move_assignable_trait> \ + : public assignment> { \ + using super = assignment>; \ + \ + public: \ + MPARK_INHERITING_CTOR(move_assignment, super) \ + using super::operator=; \ + \ + move_assignment(const move_assignment &) = default; \ + move_assignment(move_assignment &&) = default; \ + ~move_assignment() = default; \ + move_assignment &operator=(const move_assignment &) = default; \ + definition \ + } + +MPARK_VARIANT_MOVE_ASSIGNMENT( + Trait::TriviallyAvailable, + move_assignment &operator=(move_assignment &&that) = default;); + +MPARK_VARIANT_MOVE_ASSIGNMENT( + Trait::Available, + move_assignment & + operator=(move_assignment &&that) noexcept( + lib::all<(std::is_nothrow_move_constructible::value && + std::is_nothrow_move_assignable::value)...>::value) { + this->generic_assign(lib::move(that)); + return *this; + }); + +MPARK_VARIANT_MOVE_ASSIGNMENT( + Trait::Unavailable, + move_assignment &operator=(move_assignment &&) = delete;); + +#undef MPARK_VARIANT_MOVE_ASSIGNMENT + +template +class copy_assignment; + +#define MPARK_VARIANT_COPY_ASSIGNMENT(copy_assignable_trait, definition) \ + template \ + class copy_assignment, copy_assignable_trait> \ + : public move_assignment> { \ + using super = move_assignment>; \ + \ + public: \ + MPARK_INHERITING_CTOR(copy_assignment, super) \ + using super::operator=; \ + \ + copy_assignment(const copy_assignment &) = default; \ + copy_assignment(copy_assignment &&) = default; \ + ~copy_assignment() = default; \ + definition copy_assignment &operator=(copy_assignment &&) = default; \ + } + +MPARK_VARIANT_COPY_ASSIGNMENT( + Trait::TriviallyAvailable, + copy_assignment &operator=(const copy_assignment &that) = default;); + +MPARK_VARIANT_COPY_ASSIGNMENT( + Trait::Available, copy_assignment &operator=(const copy_assignment &that) { + this->generic_assign(that); + return *this; + }); + +MPARK_VARIANT_COPY_ASSIGNMENT( + Trait::Unavailable, + copy_assignment &operator=(const copy_assignment &) = delete;); + +#undef MPARK_VARIANT_COPY_ASSIGNMENT + +template +class impl : public copy_assignment> { + using super = copy_assignment>; + + public: + MPARK_INHERITING_CTOR(impl, super) + using super::operator=; + + template + inline void assign(Arg &&arg) { + this->assign_alt(access::base::get_alt(*this), lib::forward(arg)); + } + + inline void swap(impl &that) { + if (this->valueless_by_exception() && that.valueless_by_exception()) { + // do nothing. + } else if (this->index() == that.index()) { + visitation::alt::visit_alt_at(this->index(), +#ifdef MPARK_GENERIC_LAMBDAS + [](auto &this_alt, auto &that_alt) { + using std::swap; + swap(this_alt.value, that_alt.value); + } +#else + swapper {} +#endif + , + *this, + that); + } else { + impl *lhs = this; + impl *rhs = lib::addressof(that); + if (lhs->move_nothrow() && !rhs->move_nothrow()) { + std::swap(lhs, rhs); + } + impl tmp(lib::move(*rhs)); +#ifdef MPARK_EXCEPTIONS + // EXTENSION: When the move construction of `lhs` into `rhs` throws + // and `tmp` is nothrow move constructible then we move `tmp` back + // into `rhs` and provide the strong exception safety guarantee. + try { + this->generic_construct(*rhs, lib::move(*lhs)); + } catch (...) { + if (tmp.move_nothrow()) { + this->generic_construct(*rhs, lib::move(tmp)); + } + throw; + } +#else + this->generic_construct(*rhs, lib::move(*lhs)); +#endif + this->generic_construct(*lhs, lib::move(tmp)); + } + } + + private: +#ifndef MPARK_GENERIC_LAMBDAS + struct swapper { + template + inline void operator()(ThisAlt &this_alt, ThatAlt &that_alt) const { + using std::swap; + swap(this_alt.value, that_alt.value); + } + }; +#endif + + inline constexpr bool move_nothrow() const { + return this->valueless_by_exception() || + lib::array{{std::is_nothrow_move_constructible< + Ts>::value...}}[this->index()]; + } +}; + +#undef MPARK_INHERITING_CTOR + +template +struct overload_leaf { + using F = lib::size_constant (*)(T); + operator F() const { return nullptr; } +}; + +template +struct overload_impl { + private: + template + struct impl; + + template + struct impl> : overload_leaf... {}; + + public: + using type = impl>; +}; + +template +using overload = typename overload_impl::type; + +template +using best_match = lib::invoke_result_t, T &&>; + +template +struct is_in_place_index : std::false_type {}; + +template +struct is_in_place_index> : std::true_type {}; + +template +struct is_in_place_type : std::false_type {}; + +template +struct is_in_place_type> : std::true_type {}; + +} // detail + +template +class variant { + static_assert(0 < sizeof...(Ts), + "variant must consist of at least one alternative."); + + static_assert(lib::all::value...>::value, + "variant can not have an array type as an alternative."); + + static_assert(lib::all::value...>::value, + "variant can not have a reference type as an alternative."); + + static_assert(lib::all::value...>::value, + "variant can not have a void type as an alternative."); + + public: + template < + typename Front = lib::type_pack_element_t<0, Ts...>, + lib::enable_if_t::value, int> = 0> + inline constexpr variant() noexcept( + std::is_nothrow_default_constructible::value) + : impl_(in_place_index_t<0>{}) {} + + variant(const variant &) = default; + variant(variant &&) = default; + + template < + typename Arg, + typename Decayed = lib::decay_t, + lib::enable_if_t::value, int> = 0, + lib::enable_if_t::value, int> = 0, + lib::enable_if_t::value, int> = 0, + std::size_t I = detail::best_match::value, + typename T = lib::type_pack_element_t, + lib::enable_if_t::value, int> = 0> + inline constexpr variant(Arg &&arg) noexcept( + std::is_nothrow_constructible::value) + : impl_(in_place_index_t{}, lib::forward(arg)) {} + + template , + lib::enable_if_t::value, int> = 0> + inline explicit constexpr variant( + in_place_index_t, + Args + &&... args) noexcept(std::is_nothrow_constructible::value) + : impl_(in_place_index_t{}, lib::forward(args)...) {} + + template < + std::size_t I, + typename Up, + typename... Args, + typename T = lib::type_pack_element_t, + lib::enable_if_t< + std::is_constructible &, Args...>::value, + int> = 0> + inline explicit constexpr variant( + in_place_index_t, + std::initializer_list il, + Args &&... args) noexcept(std:: + is_nothrow_constructible< + T, + std::initializer_list &, + Args...>::value) + : impl_(in_place_index_t{}, il, lib::forward(args)...) {} + + template ::value, + lib::enable_if_t::value, int> = 0> + inline explicit constexpr variant( + in_place_type_t, + Args + &&... args) noexcept(std::is_nothrow_constructible::value) + : impl_(in_place_index_t{}, lib::forward(args)...) {} + + template < + typename T, + typename Up, + typename... Args, + std::size_t I = detail::find_index_sfinae::value, + lib::enable_if_t< + std::is_constructible &, Args...>::value, + int> = 0> + inline explicit constexpr variant( + in_place_type_t, + std::initializer_list il, + Args &&... args) noexcept(std:: + is_nothrow_constructible< + T, + std::initializer_list &, + Args...>::value) + : impl_(in_place_index_t{}, il, lib::forward(args)...) {} + + ~variant() = default; + + variant &operator=(const variant &) = default; + variant &operator=(variant &&) = default; + + template , variant>::value, + int> = 0, + std::size_t I = detail::best_match::value, + typename T = lib::type_pack_element_t, + lib::enable_if_t<(std::is_assignable::value && + std::is_constructible::value), + int> = 0> + inline variant &operator=(Arg &&arg) noexcept( + (std::is_nothrow_assignable::value && + std::is_nothrow_constructible::value)) { + impl_.template assign(lib::forward(arg)); + return *this; + } + + template , + lib::enable_if_t::value, int> = 0> + inline T &emplace(Args &&... args) { + return impl_.template emplace(lib::forward(args)...); + } + + template < + std::size_t I, + typename Up, + typename... Args, + typename T = lib::type_pack_element_t, + lib::enable_if_t< + std::is_constructible &, Args...>::value, + int> = 0> + inline T &emplace(std::initializer_list il, Args &&... args) { + return impl_.template emplace(il, lib::forward(args)...); + } + + template ::value, + lib::enable_if_t::value, int> = 0> + inline T &emplace(Args &&... args) { + return impl_.template emplace(lib::forward(args)...); + } + + template < + typename T, + typename Up, + typename... Args, + std::size_t I = detail::find_index_sfinae::value, + lib::enable_if_t< + std::is_constructible &, Args...>::value, + int> = 0> + inline T &emplace(std::initializer_list il, Args &&... args) { + return impl_.template emplace(il, lib::forward(args)...); + } + + inline constexpr bool valueless_by_exception() const noexcept { + return impl_.valueless_by_exception(); + } + + inline constexpr std::size_t index() const noexcept { return impl_.index(); } + + template , + Dummy>::value && + lib::dependent_type, + Dummy>::value)...>::value, + int> = 0> + inline void swap(variant &that) noexcept( + lib::all<(std::is_nothrow_move_constructible::value && + lib::is_nothrow_swappable::value)...>::value) { + impl_.swap(that.impl_); + } + + private: + detail::impl impl_; + + friend struct detail::access::variant; + friend struct detail::visitation::variant; +}; + +template +inline constexpr bool holds_alternative(const variant &v) noexcept { + return v.index() == I; +} + +template +inline constexpr bool holds_alternative(const variant &v) noexcept { + return holds_alternative::value>(v); +} + +namespace detail { +template +struct generic_get_impl { + constexpr generic_get_impl(int) noexcept {} + + constexpr AUTO_REFREF operator()(V &&v) const + AUTO_REFREF_RETURN(access::variant::get_alt(lib::forward(v)).value) +}; + +template +inline constexpr AUTO_REFREF generic_get(V &&v) + AUTO_REFREF_RETURN(generic_get_impl(holds_alternative(v) + ? 0 + : (throw_bad_variant_access(), + 0))(lib::forward(v))) +} // namespace detail + +template +inline constexpr variant_alternative_t> &get( + variant &v) { + return detail::generic_get(v); +} + +template +inline constexpr variant_alternative_t> &&get( + variant &&v) { + return detail::generic_get(lib::move(v)); +} + +template +inline constexpr const variant_alternative_t> &get( + const variant &v) { + return detail::generic_get(v); +} + +template +inline constexpr const variant_alternative_t> &&get( + const variant &&v) { + return detail::generic_get(lib::move(v)); +} + +template +inline constexpr T &get(variant &v) { + return get::value>(v); +} + +template +inline constexpr T &&get(variant &&v) { + return get::value>(lib::move(v)); +} + +template +inline constexpr const T &get(const variant &v) { + return get::value>(v); +} + +template +inline constexpr const T &&get(const variant &&v) { + return get::value>(lib::move(v)); +} + +namespace detail { + +template +inline constexpr /* auto * */ AUTO generic_get_if(V *v) noexcept AUTO_RETURN( + v &&holds_alternative(*v) + ? lib::addressof(access::variant::get_alt(*v).value) + : nullptr) + +} // namespace detail + +template +inline constexpr lib::add_pointer_t>> +get_if(variant *v) noexcept { + return detail::generic_get_if(v); +} + +template +inline constexpr lib::add_pointer_t< + const variant_alternative_t>> +get_if(const variant *v) noexcept { + return detail::generic_get_if(v); +} + +template +inline constexpr lib::add_pointer_t get_if(variant *v) noexcept { + return get_if::value>(v); +} + +template +inline constexpr lib::add_pointer_t get_if( + const variant *v) noexcept { + return get_if::value>(v); +} + +namespace detail { +template +struct convert_to_bool { + template + inline constexpr bool operator()(Lhs &&lhs, Rhs &&rhs) const { + static_assert( + std::is_convertible, bool>::value, + "relational operators must return a type" + " implicitly convertible to bool"); + return lib::invoke(RelOp{}, lib::forward(lhs), lib::forward(rhs)); + } +}; +} // namespace detail + +template +inline constexpr bool operator==(const variant &lhs, + const variant &rhs) { + using detail::visitation::variant; + using equal_to = detail::convert_to_bool; +#ifdef MPARK_CPP14_CONSTEXPR + if (lhs.index() != rhs.index()) return false; + if (lhs.valueless_by_exception()) return true; + return variant::visit_value_at(lhs.index(), equal_to{}, lhs, rhs); +#else + return lhs.index() == rhs.index() && + (lhs.valueless_by_exception() || + variant::visit_value_at(lhs.index(), equal_to{}, lhs, rhs)); +#endif +} + +template +inline constexpr bool operator!=(const variant &lhs, + const variant &rhs) { + using detail::visitation::variant; + using not_equal_to = detail::convert_to_bool; +#ifdef MPARK_CPP14_CONSTEXPR + if (lhs.index() != rhs.index()) return true; + if (lhs.valueless_by_exception()) return false; + return variant::visit_value_at(lhs.index(), not_equal_to{}, lhs, rhs); +#else + return lhs.index() != rhs.index() || + (!lhs.valueless_by_exception() && + variant::visit_value_at(lhs.index(), not_equal_to{}, lhs, rhs)); +#endif +} + +template +inline constexpr bool operator<(const variant &lhs, + const variant &rhs) { + using detail::visitation::variant; + using less = detail::convert_to_bool; +#ifdef MPARK_CPP14_CONSTEXPR + if (rhs.valueless_by_exception()) return false; + if (lhs.valueless_by_exception()) return true; + if (lhs.index() < rhs.index()) return true; + if (lhs.index() > rhs.index()) return false; + return variant::visit_value_at(lhs.index(), less{}, lhs, rhs); +#else + return !rhs.valueless_by_exception() && + (lhs.valueless_by_exception() || lhs.index() < rhs.index() || + (lhs.index() == rhs.index() && + variant::visit_value_at(lhs.index(), less{}, lhs, rhs))); +#endif +} + +template +inline constexpr bool operator>(const variant &lhs, + const variant &rhs) { + using detail::visitation::variant; + using greater = detail::convert_to_bool; +#ifdef MPARK_CPP14_CONSTEXPR + if (lhs.valueless_by_exception()) return false; + if (rhs.valueless_by_exception()) return true; + if (lhs.index() > rhs.index()) return true; + if (lhs.index() < rhs.index()) return false; + return variant::visit_value_at(lhs.index(), greater{}, lhs, rhs); +#else + return !lhs.valueless_by_exception() && + (rhs.valueless_by_exception() || lhs.index() > rhs.index() || + (lhs.index() == rhs.index() && + variant::visit_value_at(lhs.index(), greater{}, lhs, rhs))); +#endif +} + +template +inline constexpr bool operator<=(const variant &lhs, + const variant &rhs) { + using detail::visitation::variant; + using less_equal = detail::convert_to_bool; +#ifdef MPARK_CPP14_CONSTEXPR + if (lhs.valueless_by_exception()) return true; + if (rhs.valueless_by_exception()) return false; + if (lhs.index() < rhs.index()) return true; + if (lhs.index() > rhs.index()) return false; + return variant::visit_value_at(lhs.index(), less_equal{}, lhs, rhs); +#else + return lhs.valueless_by_exception() || + (!rhs.valueless_by_exception() && + (lhs.index() < rhs.index() || + (lhs.index() == rhs.index() && + variant::visit_value_at(lhs.index(), less_equal{}, lhs, rhs)))); +#endif +} + +template +inline constexpr bool operator>=(const variant &lhs, + const variant &rhs) { + using detail::visitation::variant; + using greater_equal = detail::convert_to_bool; +#ifdef MPARK_CPP14_CONSTEXPR + if (rhs.valueless_by_exception()) return true; + if (lhs.valueless_by_exception()) return false; + if (lhs.index() > rhs.index()) return true; + if (lhs.index() < rhs.index()) return false; + return variant::visit_value_at(lhs.index(), greater_equal{}, lhs, rhs); +#else + return rhs.valueless_by_exception() || + (!lhs.valueless_by_exception() && + (lhs.index() > rhs.index() || + (lhs.index() == rhs.index() && + variant::visit_value_at(lhs.index(), greater_equal{}, lhs, rhs)))); +#endif +} + +struct monostate {}; + +inline constexpr bool operator<(monostate, monostate) noexcept { return false; } + +inline constexpr bool operator>(monostate, monostate) noexcept { return false; } + +inline constexpr bool operator<=(monostate, monostate) noexcept { return true; } + +inline constexpr bool operator>=(monostate, monostate) noexcept { return true; } + +inline constexpr bool operator==(monostate, monostate) noexcept { return true; } + +inline constexpr bool operator!=(monostate, monostate) noexcept { + return false; +} + +#ifdef MPARK_CPP14_CONSTEXPR +namespace detail { + +inline constexpr bool all(std::initializer_list bs) { + for (bool b : bs) { + if (!b) { + return false; + } + } + return true; +} + +} // namespace detail + +template +inline constexpr decltype(auto) visit(Visitor &&visitor, Vs &&... vs) { + return (detail::all( + lib::array{!vs.valueless_by_exception()...}) + ? (void)0 + : throw_bad_variant_access()), + detail::visitation::variant::visit_value( + lib::forward(visitor), lib::forward(vs)...); +} +#else +namespace detail { + +template +inline constexpr bool all_impl(const lib::array &bs, std::size_t idx) { + return idx >= N || (bs[idx] && all_impl(bs, idx + 1)); +} + +template +inline constexpr bool all(const lib::array &bs) { + return all_impl(bs, 0); +} + +} // namespace detail + +template +inline constexpr DECLTYPE_AUTO visit(Visitor &&visitor, Vs &&... vs) + DECLTYPE_AUTO_RETURN( + (detail::all(lib::array{ + {!vs.valueless_by_exception()...}}) + ? (void)0 + : throw_bad_variant_access()), + detail::visitation::variant::visit_value(lib::forward(visitor), + lib::forward(vs)...)) +#endif + +template +inline auto swap(variant &lhs, + variant &rhs) noexcept(noexcept(lhs.swap(rhs))) + -> decltype(lhs.swap(rhs)) { + lhs.swap(rhs); +} + +namespace detail { + +template +using enabled_type = T; + +namespace hash { + +template +constexpr bool meets_requirements() noexcept { + return std::is_copy_constructible::value && + std::is_move_constructible::value && + lib::is_invocable_r::value; +} + +template +constexpr bool is_enabled() noexcept { + using H = std::hash; + return meets_requirements() && + std::is_default_constructible::value && + std::is_copy_assignable::value && std::is_move_assignable::value; +} + +} // namespace hash + +} // namespace detail + +#undef AUTO +#undef AUTO_RETURN + +#undef AUTO_REFREF +#undef AUTO_REFREF_RETURN + +#undef DECLTYPE_AUTO +#undef DECLTYPE_AUTO_RETURN + +} // namespace paddle + +namespace std { + +template +struct hash, + paddle::lib::enable_if_t>()...>::value>>> { + using argument_type = paddle::variant; + using result_type = std::size_t; + + inline result_type operator()(const argument_type &v) const { + using paddle::detail::visitation::variant; + std::size_t result = + v.valueless_by_exception() + ? 299792458 // Random value chosen by the universe upon creation + : variant::visit_alt( +#ifdef MPARK_GENERIC_LAMBDAS + [](const auto &alt) { + using alt_type = paddle::lib::decay_t; + using value_type = paddle::lib::remove_const_t< + typename alt_type::value_type>; + return hash{}(alt.value); + } +#else + hasher {} +#endif + , + v); + return hash_combine(result, hash{}(v.index())); + } + + private: +#ifndef MPARK_GENERIC_LAMBDAS + struct hasher { + template + inline std::size_t operator()(const Alt &alt) const { + using alt_type = paddle::lib::decay_t; + using value_type = + paddle::lib::remove_const_t; + return hash{}(alt.value); + } + }; +#endif + + static std::size_t hash_combine(std::size_t lhs, std::size_t rhs) { + return lhs ^= rhs + 0x9e3779b9 + (lhs << 6) + (lhs >> 2); + } +}; + +template <> +struct hash { + using argument_type = paddle::monostate; + using result_type = std::size_t; + + inline result_type operator()(const argument_type &) const noexcept { + return 66740831; // return a fundamentally attractive random value. + } +}; + +} // namespace std diff --git a/python/paddle/distributed/collective.py b/python/paddle/distributed/collective.py index b92b2a3c15dec..b2d146297de8a 100644 --- a/python/paddle/distributed/collective.py +++ b/python/paddle/distributed/collective.py @@ -350,18 +350,19 @@ def new_group(ranks=None, backend=None): global _default_group_name gid = _new_ring_id() group_name = _default_group_name + str(gid) - global_group = _get_default_group() - global_rank = global_group.rank - global_ranks = global_group.ranks - backend = _default_backend if backend is None else backend - if ranks is None: - ranks = global_ranks - assert len(ranks) <= len(global_ranks), ( - "Size of new group must be less than or " - "equal to that of the default global group.") + if ranks is None or len(ranks) > 1: + global_group = _get_default_group() + global_rank = global_group.rank + global_ranks = global_group.ranks + backend = _default_backend if backend is None else backend + if ranks is None: + ranks = global_ranks + assert len(ranks) <= len(global_ranks), ( + "Size of new group must be less than or " + "equal to that of the default global group.") size = len(ranks) ranks = sorted(ranks) - if global_rank in ranks and size > 1: + if size > 1 and global_rank in ranks: rank = ranks.index(global_rank) pg = _new_process_group_impl( backend, @@ -642,6 +643,8 @@ def all_reduce(tensor, op=ReduceOp.SUM, group=None, use_calc_stream=True): op_type = core.ReduceOp.MAX elif op == ReduceOp.MIN: op_type = core.ReduceOp.MIN + elif op == ReduceOp.PROD: + op_type = core.ReduceOp.PRODUCT else: raise ValueError("Unknown reduce_op type for allreduce.") group = _get_default_group() if group is None else group @@ -744,6 +747,8 @@ def reduce(tensor, dst, op=ReduceOp.SUM, group=None, use_calc_stream=True): op_type = core.ReduceOp.MAX elif op == ReduceOp.MIN: op_type = core.ReduceOp.MIN + elif op == ReduceOp.PROD: + op_type = core.ReduceOp.PRODUCT else: raise ValueError("Unknown reduce_op type for reduce.") group = _get_default_group() if group is None else group diff --git a/python/paddle/distributed/fleet/__init__.py b/python/paddle/distributed/fleet/__init__.py index 3186df7db581a..ef0fff8283361 100644 --- a/python/paddle/distributed/fleet/__init__.py +++ b/python/paddle/distributed/fleet/__init__.py @@ -77,6 +77,7 @@ distributed_optimizer = fleet.distributed_optimizer save_inference_model = fleet.save_inference_model save_persistables = fleet.save_persistables +save_cache_model = fleet.save_cache_model load_model = fleet.load_model minimize = fleet.minimize distributed_model = fleet.distributed_model diff --git a/python/paddle/distributed/fleet/base/fleet_base.py b/python/paddle/distributed/fleet/base/fleet_base.py index 4e975e74bdb14..a1c967ab0639c 100755 --- a/python/paddle/distributed/fleet/base/fleet_base.py +++ b/python/paddle/distributed/fleet/base/fleet_base.py @@ -869,6 +869,11 @@ def save_persistables(self, executor, dirname, main_program=None, mode=0): self._runtime_handle._save_persistables(executor, dirname, main_program, mode) + @is_non_distributed_check + @inited_runtime_handler + def save_cache_model(self, dirname, **configs): + return self._runtime_handle._save_cache_model(dirname, **configs) + def shrink(self, threshold=None): self._runtime_handle._shrink(threshold) diff --git a/python/paddle/distributed/parallel.py b/python/paddle/distributed/parallel.py index f0365cab8c896..53d35a251c8c8 100644 --- a/python/paddle/distributed/parallel.py +++ b/python/paddle/distributed/parallel.py @@ -219,8 +219,9 @@ def train(): "required to create a process group.") master_addr = os.getenv("MASTER_ADDR", None) master_port = os.getenv("MASTER_PORT", None) - endpoints = None - if not master_addr or not master_port: + endpoints = ":".join( + [master_addr, master_port]) if master_addr and master_port else None + if endpoints is None: endpoints = os.getenv("PADDLE_MASTER", None) if endpoints is None: endpoints = os.getenv("PADDLE_TRAINER_ENDPOINTS").split(',')[0] diff --git a/python/paddle/distributed/ps/the_one_ps.py b/python/paddle/distributed/ps/the_one_ps.py index 5be739785ff44..c6df7559a22e8 100755 --- a/python/paddle/distributed/ps/the_one_ps.py +++ b/python/paddle/distributed/ps/the_one_ps.py @@ -1315,6 +1315,30 @@ def _save_inference_model(self, *args, **kwargs): def _save_persistables(self, *args, **kwargs): self._ps_inference_save_persistables(*args, **kwargs) + def _save_cache_model(self, dirname, **kwargs): + mode = kwargs.get("mode", 0) + table_id = kwargs.get("table_id", 0) + self._worker.client_flush() + fleet.util.barrier() + cache_threshold = 0.0 + + if self.role_maker._is_first_worker(): + cache_threshold = self._worker.get_cache_threshold(table_id) + #check cache threshold right or not + fleet.util.barrier() + + if self.role_maker._is_first_worker(): + self._worker.cache_shuffle(table_id, dirname, mode, cache_threshold) + + fleet.util.barrier() + + feasign_num = -1 + if self.role_maker._is_first_worker(): + feasign_num = self._worker.save_cache(table_id, dirname, mode) + + fleet.util.barrier() + return feasign_num + def _load_sparse_params(self, dirname, context, main_program, mode): distributed_varnames = get_sparse_tablenames(self.origin_main_programs, True) diff --git a/python/paddle/distribution/categorical.py b/python/paddle/distribution/categorical.py index b181a25fbcee1..97a3df490b1d0 100644 --- a/python/paddle/distribution/categorical.py +++ b/python/paddle/distribution/categorical.py @@ -115,6 +115,8 @@ def __init__(self, logits, name=None): self.logits = self._to_tensor(logits)[0] if self.dtype != convert_dtype(self.logits.dtype): self.logits = tensor.cast(self.logits, dtype=self.dtype) + dist_sum = paddle.sum(self.logits, axis=-1, keepdim=True) + self._prob = self.logits / dist_sum def sample(self, shape): """Generate samples of the specified shape. @@ -297,42 +299,21 @@ def probs(self, value): """ name = self.name + '_probs' - - dist_sum = paddle.sum(self.logits, axis=-1, keepdim=True) - prob = self.logits / dist_sum - - shape = list(prob.shape) - value_shape = list(value.shape) - if len(shape) == 1: - num_value_in_one_dist = np.prod(value_shape) - index_value = paddle.reshape(value, [num_value_in_one_dist, 1]) - index = index_value + if len(self._prob.shape) == 1: # batch_shape is empty + return paddle.gather( + self._prob, value.reshape( + [-1], name=name), name=name).reshape( + value.shape, name=name) else: - num_dist = np.prod(shape[:-1]) - num_value_in_one_dist = value_shape[-1] - prob = paddle.reshape(prob, [num_dist, shape[-1]]) - if len(value_shape) == 1: - value = nn.expand(value, [num_dist]) - value_shape = shape[:-1] + value_shape - index_value = paddle.reshape(value, [num_dist, -1, 1]) - if shape[:-1] != value_shape[:-1]: - raise ValueError( - "shape of value {} must match shape of logits {}".format( - str(value_shape[:-1]), str(shape[:-1]))) - - index_prefix = paddle.unsqueeze( - arange( - num_dist, dtype=index_value.dtype), axis=-1) - index_prefix = nn.expand(index_prefix, [1, num_value_in_one_dist]) - index_prefix = paddle.unsqueeze(index_prefix, axis=-1) - - if index_value.dtype != index_prefix.dtype: - tensor.cast(index_prefix, dtype=index_value.dtype) - index = concat([index_prefix, index_value], axis=-1) - - # value is the category index to search for the corresponding probability. - select_prob = gather_nd(prob, index) - return paddle.reshape(select_prob, value_shape, name=name) + if len(value.shape) == 1: + return paddle.take_along_axis( + self._prob, + paddle.reshape( + value, (len(self._prob.shape) - 1) * [1] + [-1], + name=name), + axis=-1) + else: + return paddle.take_along_axis(self._prob, value, axis=-1) def log_prob(self, value): """Log probabilities of the given category. Refer to ``probs`` method. diff --git a/python/paddle/distribution/multinomial.py b/python/paddle/distribution/multinomial.py index c4110040fd192..837eb53eab1ea 100644 --- a/python/paddle/distribution/multinomial.py +++ b/python/paddle/distribution/multinomial.py @@ -16,6 +16,10 @@ import paddle from paddle.distribution import categorical, distribution +try: + from collections.abc import Iterable +except: + from collections import Iterable class Multinomial(distribution.Distribution): @@ -138,7 +142,7 @@ def sample(self, shape=()): Args: sample_shape (tuple, optional): [description]. Defaults to (). """ - if not isinstance(shape, collections.Iterable): + if not isinstance(shape, Iterable): raise TypeError('sample shape must be Iterable object.') samples = self._categorical.sample([self.total_count, ] + list(shape)) diff --git a/python/paddle/fluid/backward.py b/python/paddle/fluid/backward.py index adce805195960..5fdbbb4d7ed18 100755 --- a/python/paddle/fluid/backward.py +++ b/python/paddle/fluid/backward.py @@ -28,6 +28,10 @@ import paddle.fluid from .data_feeder import check_type import warnings +try: + from collections.abc import Sequence +except: + from collections import Sequence __all__ = [ 'append_backward', 'gradients', @@ -1722,7 +1726,7 @@ def append_backward(loss, def _as_list(x): if x is None: return [] - return list(x) if isinstance(x, collections.Sequence) else [x] + return list(x) if isinstance(x, Sequence) else [x] def _is_ancestor_block(ancestor_block, block): @@ -2021,7 +2025,6 @@ def calc_gradient(targets, inputs, target_gradients=None, no_grad_set=None): @framework.static_only def gradients(targets, inputs, target_gradients=None, no_grad_set=None): """ - :api_attr: Static Graph Backpropagate the gradients of targets to inputs. @@ -2042,8 +2045,9 @@ def gradients(targets, inputs, target_gradients=None, no_grad_set=None): will be None. Examples: + .. code-block:: python - + :name: code-example import paddle import paddle.nn.functional as F diff --git a/python/paddle/fluid/clip.py b/python/paddle/fluid/clip.py index 0ba980c3e9233..172929608dbde 100644 --- a/python/paddle/fluid/clip.py +++ b/python/paddle/fluid/clip.py @@ -468,10 +468,15 @@ class ClipGradByGlobalNorm(ClipGradBase): sdg.step() """ - def __init__(self, clip_norm, group_name="default_group"): + def __init__(self, + clip_norm, + group_name="default_group", + auto_skip_clip=False): super(ClipGradByGlobalNorm, self).__init__() self.clip_norm = float(clip_norm) self.group_name = group_name + assert isinstance(auto_skip_clip, bool) + self.auto_skip_clip = auto_skip_clip def __str__(self): return "Gradient Clip By GlobalNorm, global_norm=%f" % (self.clip_norm) @@ -524,14 +529,19 @@ def _dygraph_clip(self, params_grads): max_global_norm = layers.fill_constant( shape=[1], dtype=global_norm_var.dtype, value=self.clip_norm) - # only when global_norm_var > max_global_norm, grad need clip need_clip = False - if global_norm_var > max_global_norm: + if not self.auto_skip_clip: # always apply clip + need_clip = True + clip_var = layers.elementwise_div( + x=max_global_norm, + y=layers.elementwise_max( + x=global_norm_var, y=max_global_norm)) + elif global_norm_var > max_global_norm: + # only when global_norm_var > max_global_norm, grad need clip need_clip = True - - if need_clip: clip_var = layers.elementwise_div( x=max_global_norm, y=global_norm_var) + for p, g in params_grads: if g is None: continue diff --git a/python/paddle/fluid/contrib/mixed_precision/amp_nn.py b/python/paddle/fluid/contrib/mixed_precision/amp_nn.py index 588eb2a29f555..c5b9b9e71f6be 100644 --- a/python/paddle/fluid/contrib/mixed_precision/amp_nn.py +++ b/python/paddle/fluid/contrib/mixed_precision/amp_nn.py @@ -129,9 +129,13 @@ def update_loss_scaling(x, 'decr_every_n_nan_or_inf': decr_every_n_nan_or_inf, 'incr_ratio': incr_ratio, 'decr_ratio': decr_ratio, - 'stop_update': stop_update } + if isinstance(stop_update, Variable): + inputs['StopUpdate'] = stop_update + else: + attrs['stop_update'] = stop_update + helper.append_op( type='update_loss_scaling', inputs=inputs, outputs=outputs, attrs=attrs) diff --git a/python/paddle/fluid/contrib/mixed_precision/decorator.py b/python/paddle/fluid/contrib/mixed_precision/decorator.py index c6e2bcb8b1a24..c3720396e1d77 100644 --- a/python/paddle/fluid/contrib/mixed_precision/decorator.py +++ b/python/paddle/fluid/contrib/mixed_precision/decorator.py @@ -432,7 +432,7 @@ def _add_dynamic_loss_scaling(self, params_grads, found_inf): self._decr_every_n_nan_or_inf, self._incr_ratio, self._decr_ratio, - stop_update=False, + stop_update=self._optimizer._get_stop_update_var(), name="update_loss_scaling") return diff --git a/python/paddle/fluid/contrib/mixed_precision/fp16_lists.py b/python/paddle/fluid/contrib/mixed_precision/fp16_lists.py index 9dba5d658dfc9..7b2546f70ad1b 100644 --- a/python/paddle/fluid/contrib/mixed_precision/fp16_lists.py +++ b/python/paddle/fluid/contrib/mixed_precision/fp16_lists.py @@ -162,6 +162,7 @@ def _update_list(self): 'split', 'fused_feedforward', 'fused_attention', + 'fused_multi_transformer', } # The set of ops that don't support fp16 calculation diff --git a/python/paddle/fluid/contrib/mixed_precision/fp16_utils.py b/python/paddle/fluid/contrib/mixed_precision/fp16_utils.py index 760e9ceb9ea2f..0100866806cdc 100644 --- a/python/paddle/fluid/contrib/mixed_precision/fp16_utils.py +++ b/python/paddle/fluid/contrib/mixed_precision/fp16_utils.py @@ -109,6 +109,8 @@ def _keep_fp32_input(op, in_name): return in_name in { 'LnScale', 'LnBias', 'Ln2Scale', 'Ln2Bias', "Ln1Scale", "Ln1Bias" } + if op_type == 'fused_multi_transformer': + return in_name in {'LnScale', 'LnBias', 'FFNLnScale', 'FFNLnBias'} return False diff --git a/python/paddle/fluid/dataloader/dataloader_iter.py b/python/paddle/fluid/dataloader/dataloader_iter.py index bbf2a4377c767..430578db51022 100644 --- a/python/paddle/fluid/dataloader/dataloader_iter.py +++ b/python/paddle/fluid/dataloader/dataloader_iter.py @@ -31,6 +31,7 @@ import paddle import paddle.profiler as profiler +from paddle.profiler.utils import in_profiler_mode from .. import core, layers from ..framework import _non_static_mode, in_dygraph_mode, _in_legacy_dygraph from ..multiprocess_utils import _set_SIGCHLD_handler, MP_STATUS_CHECK_INTERVAL, CleanupFuncRegistrar @@ -252,10 +253,11 @@ def _thread_loop(self, legacy_expected_place): self._exit_thread_expectedly() def __next__(self): - trace_event = profiler.RecordEvent( - name="_DataLoaderIterSingleProcess", - event_type=profiler.TracerEventType.Dataloader) - trace_event.begin() + if in_profiler_mode(): + trace_event = profiler.RecordEvent( + name="_DataLoaderIterSingleProcess", + event_type=profiler.TracerEventType.Dataloader) + trace_event.begin() try: benchmark().check_if_need_record(self) benchmark().before_reader() @@ -294,7 +296,8 @@ def __next__(self): self._try_shutdown_all() six.reraise(*sys.exc_info()) finally: - trace_event.end() + if in_profiler_mode(): + trace_event.end() def _shutdown_thread(self): if self._thread: @@ -708,10 +711,11 @@ def _shutdown_on_exit(self): self._try_shutdown_all(1) def __next__(self): - trace_event = profiler.RecordEvent( - name="_DataLoaderIterMultiProcess", - event_type=profiler.TracerEventType.Dataloader) - trace_event.begin() + if in_profiler_mode(): + trace_event = profiler.RecordEvent( + name="_DataLoaderIterMultiProcess", + event_type=profiler.TracerEventType.Dataloader) + trace_event.begin() try: benchmark().check_if_need_record(self) benchmark().before_reader() @@ -765,7 +769,8 @@ def __next__(self): self._try_shutdown_all() six.reraise(*sys.exc_info()) finally: - trace_event.end() + if in_profiler_mode(): + trace_event.end() # python2 compatibility def next(self): diff --git a/python/paddle/fluid/dygraph/amp/auto_cast.py b/python/paddle/fluid/dygraph/amp/auto_cast.py index 8703312d07437..6ca0c441b166f 100644 --- a/python/paddle/fluid/dygraph/amp/auto_cast.py +++ b/python/paddle/fluid/dygraph/amp/auto_cast.py @@ -277,9 +277,10 @@ def amp_guard(enable=True, if enable and not (tracer._expected_place.is_gpu_place() or tracer._expected_place.is_xpu_place() or tracer._expected_place.is_mlu_place() or - tracer._expected_place.is_npu_place()): + tracer._expected_place.is_npu_place() or + tracer._expected_place.is_custom_place()): warnings.warn( - 'amp_guard can only be enabled on CUDAPlace, XPUPlace, MLUPlace, and NPUPlace, current place is %s, so it makes no effect.' + 'amp_guard can only be enabled on CUDAPlace, XPUPlace, MLUPlace, NPUPlace, and CustomPlace, current place is %s, so it makes no effect.' % tracer._expected_place) enable = False # For npu: @@ -294,6 +295,10 @@ def amp_guard(enable=True, if tracer._expected_place.is_mlu_place() and (dtype == 'bfloat16'): warnings.warn('MLUPlace only support float16 amp.') enable = False + # For custom device: + if tracer._expected_place.is_custom_place() and (dtype == 'bfloat16'): + warnings.warn('CustomPlace only support float16 amp.') + enable = False # For gpu float16: Compute Capability should >= 7. # For gpu bfloat16: Compute Capability should >= 8 & CUDA Version should >= 11. if tracer._expected_place.is_gpu_place(): diff --git a/python/paddle/fluid/dygraph/amp/loss_scaler.py b/python/paddle/fluid/dygraph/amp/loss_scaler.py index c57290861942b..df79b5ab5e482 100644 --- a/python/paddle/fluid/dygraph/amp/loss_scaler.py +++ b/python/paddle/fluid/dygraph/amp/loss_scaler.py @@ -107,9 +107,10 @@ def __init__(self, if enable and not (tracer._expected_place.is_gpu_place() or tracer._expected_place.is_xpu_place() or tracer._expected_place.is_mlu_place() or - tracer._expected_place.is_npu_place()): + tracer._expected_place.is_npu_place() or + tracer._expected_place.is_custom_place()): warnings.warn( - 'AmpScaler can only be enabled on CUDAPlace, XPUPlace, MLUPlace and NPUPlace, current place is %s, so it makes no effect.' + 'AmpScaler can only be enabled on CUDAPlace, XPUPlace, MLUPlace, NPUPlace and CustomPlace, current place is %s, so it makes no effect.' % tracer._expected_place) enable = False diff --git a/python/paddle/fluid/dygraph/dygraph_to_static/origin_info.py b/python/paddle/fluid/dygraph/dygraph_to_static/origin_info.py index 0670c048c5e26..60043c42121bd 100644 --- a/python/paddle/fluid/dygraph/dygraph_to_static/origin_info.py +++ b/python/paddle/fluid/dygraph/dygraph_to_static/origin_info.py @@ -21,6 +21,10 @@ from paddle.fluid import core from paddle.fluid.dygraph.dygraph_to_static.utils import unwrap from paddle.fluid.framework import Program +try: + from collections.abc import Sequence +except: + from collections import Sequence # NOTE(liym27): Please use `getattr(ast_node, ORIGI_INFO)` instead of . operation to get the original information of ast node. ORIGI_INFO = "Original information of source code for ast node." @@ -214,7 +218,7 @@ def ast_walk(transformed_node, static_node): def _as_list(x): if x is None: return [] - return list(x) if isinstance(x, collections.Sequence) else [x] + return list(x) if isinstance(x, Sequence) else [x] transformed_node_list = _as_list(transformed_node) static_node_list = _as_list(static_node) diff --git a/python/paddle/fluid/dygraph/layers.py b/python/paddle/fluid/dygraph/layers.py index 41c1a0aa5808e..088fed03c3595 100644 --- a/python/paddle/fluid/dygraph/layers.py +++ b/python/paddle/fluid/dygraph/layers.py @@ -26,6 +26,7 @@ import paddle import paddle.profiler as profiler +from paddle.profiler.utils import in_profiler_mode from . import parallel_helper from .. import unique_name @@ -906,8 +907,11 @@ def _dygraph_call_func(self, *inputs, **kwargs): self._built = True - with profiler.RecordEvent(self.full_name(), - profiler.TracerEventType.Forward): + if in_profiler_mode(): + with profiler.RecordEvent(self.full_name(), + profiler.TracerEventType.Forward): + outputs = self.forward(*inputs, **kwargs) + else: outputs = self.forward(*inputs, **kwargs) for forward_post_hook in self._forward_post_hooks.values(): @@ -919,7 +923,7 @@ def _dygraph_call_func(self, *inputs, **kwargs): def __call__(self, *inputs, **kwargs): if (not in_declarative_mode()) and (not self._forward_pre_hooks) \ - and (not self._forward_post_hooks) and (not self._built) and in_dygraph_mode(): + and (not self._forward_post_hooks) and (not self._built) and in_dygraph_mode() and (not in_profiler_mode()): self._build_once(*inputs, **kwargs) return self.forward(*inputs, **kwargs) else: diff --git a/python/paddle/fluid/dygraph/math_op_patch.py b/python/paddle/fluid/dygraph/math_op_patch.py index 8ce56d5a92686..8a19be640a7ff 100644 --- a/python/paddle/fluid/dygraph/math_op_patch.py +++ b/python/paddle/fluid/dygraph/math_op_patch.py @@ -222,7 +222,9 @@ def __impl__(self, other_var): # so the calculation result here and the calculation result of numpy are # different after 6 decimal point. If necessary, we can also use float64 here. # torch's behavior here is consistent with ours - if op_type == 'elementwise_div' and self.dtype in _supported_int_dtype_: + if (op_type == "final_state_divide" or + op_type == "elementwise_div" + ) and self.dtype in _supported_int_dtype_: self = astype(self, 'float32') # here use `scale` replace `elementwise` to get better performance # but only +, -, *, / can use this method @@ -277,7 +279,8 @@ def __impl__(self, other_var): self = other_var other_var = tmp - if op_type == 'elementwise_div' and self.dtype in _supported_int_dtype_: + if (op_type == "final_state_divide" or op_type == "elementwise_div" + ) and self.dtype in _supported_int_dtype_: self = astype(self, 'float32') other_var = astype(other_var, 'float32') diff --git a/python/paddle/fluid/dygraph/varbase_patch_methods.py b/python/paddle/fluid/dygraph/varbase_patch_methods.py index db6af87635ccb..a93facbc34a5b 100644 --- a/python/paddle/fluid/dygraph/varbase_patch_methods.py +++ b/python/paddle/fluid/dygraph/varbase_patch_methods.py @@ -30,6 +30,7 @@ from paddle.fluid.data_feeder import convert_dtype, _PADDLE_DTYPE_2_NUMPY_DTYPE import paddle.utils.deprecated as deprecated import paddle.profiler as profiler +from paddle.profiler.utils import in_profiler_mode from paddle import _C_ops _grad_scalar = None @@ -247,9 +248,10 @@ def backward(self, grad_tensor=None, retain_graph=False): """ if framework._non_static_mode(): - record_event = profiler.RecordEvent( - "Gradient Backward", profiler.TracerEventType.Backward) - record_event.begin() + if in_profiler_mode(): + record_event = profiler.RecordEvent( + "Gradient Backward", profiler.TracerEventType.Backward) + record_event.begin() if grad_tensor is not None: if framework._in_eager_mode_: assert isinstance( @@ -289,7 +291,8 @@ def backward(self, grad_tensor=None, retain_graph=False): core.dygraph_run_backward([self], [grad_tensor], retain_graph, framework._dygraph_tracer()) - record_event.end() + if in_profiler_mode(): + record_event.end() else: raise ValueError( "Variable.backward() is only available in DyGraph mode") diff --git a/python/paddle/fluid/executor.py b/python/paddle/fluid/executor.py index 86b0d6560c927..c6ff3a583d6a3 100644 --- a/python/paddle/fluid/executor.py +++ b/python/paddle/fluid/executor.py @@ -75,7 +75,6 @@ def _switch_scope(scope): @signature_safe_contextmanager def scope_guard(scope): """ - :api_attr: Static Graph This function switches scope through python `with` statement. Scope records the mapping between variable names and variables ( :ref:`api_guide_Variable` ), @@ -94,6 +93,7 @@ def scope_guard(scope): None Examples: + .. code-block:: python import paddle @@ -1386,7 +1386,8 @@ def _run_impl(self, program, feed, fetch_list, feed_var_name, def _can_use_interpreter_core(program, place): if core.is_compiled_with_npu() or core.is_compiled_with_xpu( - ) or core.is_compiled_with_mlu() or core.is_compiled_with_ipu(): + ) or core.is_compiled_with_mlu() or core.is_compiled_with_ipu( + ) or isinstance(place, core.CustomPlace): return False compiled = isinstance(program, compiler.CompiledProgram) diff --git a/python/paddle/fluid/framework.py b/python/paddle/fluid/framework.py index 314a502a3cbef..16a5e25472557 100644 --- a/python/paddle/fluid/framework.py +++ b/python/paddle/fluid/framework.py @@ -729,7 +729,7 @@ def is_compiled_with_rocm(): def cuda_places(device_ids=None): """ - **Note**: + Note: For multi-card tasks, please use `FLAGS_selected_gpus` environment variable to set the visible GPU device. The next version will fix the problem with `CUDA_VISIBLE_DEVICES` environment variable. @@ -754,6 +754,7 @@ def cuda_places(device_ids=None): list of paddle.CUDAPlace: Created GPU place list. Examples: + .. code-block:: python import paddle @@ -874,6 +875,7 @@ def cpu_places(device_count=None): list of paddle.CPUPlace: Created list of CPU places. Examples: + .. code-block:: python import paddle @@ -993,7 +995,6 @@ def name(self): @signature_safe_contextmanager def name_scope(prefix=None): """ - :api_attr: Static Graph Generate hierarchical name prefix for the operators in Static Graph. @@ -1006,6 +1007,7 @@ def name_scope(prefix=None): prefix(str, optional): prefix. Default is none. Examples: + .. code-block:: python import paddle @@ -2861,8 +2863,22 @@ def _to_readable_code(self, skip_op_callstack=True): attrs_str += ", " continue + # it is bytes of serialized protobuf + if self.type == 'cinn_launch' and name == 'compilation_key': + # value = core.get_readable_comile_key(self.desc) + v = self.desc.attr(name) + prog = Program() + prog = prog.parse_from_string(v) + s = prog._to_readable_code() + lines = s.split('\n') + value = '\n'.join([' ' + line for line in lines]) + value = '\n' + value + else: + value = self.desc.attr(name) + a = "{name} = {value}".format( - name=name, type=attr_type, value=self.desc.attr(name)) + name=name, type=attr_type, value=value) + attrs_str += a if i != len(attr_names) - 1: attrs_str += ", " @@ -6916,8 +6932,9 @@ def switch_device(device): @signature_safe_contextmanager def device_guard(device=None): """ - **Notes**: - **The API only supports static mode.** + + Note: + The API only supports static mode. A context manager that specifies the device on which the OP will be placed. @@ -6931,8 +6948,10 @@ def device_guard(device=None): assigned devices. Examples: + .. code-block:: python - + + # required: gpu import paddle paddle.enable_static() diff --git a/python/paddle/fluid/incubate/fleet/parameter_server/pslib/__init__.py b/python/paddle/fluid/incubate/fleet/parameter_server/pslib/__init__.py index 8d803c0d5bd7d..40ff41fe89f47 100644 --- a/python/paddle/fluid/incubate/fleet/parameter_server/pslib/__init__.py +++ b/python/paddle/fluid/incubate/fleet/parameter_server/pslib/__init__.py @@ -1139,10 +1139,11 @@ def minimize(self, from paddle.fluid.transpiler.collective import MultiThread # check start program if program_mode not in [ - "all_reduce", "fuse_all_reduce", "all_gather" + "all_reduce", "fuse_all_reduce", "all_gather", + "all_reduce_xpu" ]: raise ValueError("You should set program_mode in [ all_reduce, \ - fuse_all_reduce, all_gather ]") + fuse_all_reduce, all_gather, all_reduce_xpu ]") env = self.get_dist_env() if not isinstance(losses, list): startup_programs = [startup_programs] diff --git a/python/paddle/fluid/initializer.py b/python/paddle/fluid/initializer.py index ba5e51c11dd65..1c8e399436625 100644 --- a/python/paddle/fluid/initializer.py +++ b/python/paddle/fluid/initializer.py @@ -353,7 +353,6 @@ def __call__(self, var, block=None): out_var = _C_ops.final_state_gaussian_random( var.shape, self._mean, self._std_dev, self._seed, out_dtype, place) - out_var._share_underline_tensor_to(var) if var.dtype in [VarDesc.VarType.FP16, VarDesc.VarType.BF16]: var_tmp = _C_ops.final_state_cast(out_var, var.dtype) diff --git a/python/paddle/fluid/io.py b/python/paddle/fluid/io.py index a48cfd9150c65..7c7f101286e24 100644 --- a/python/paddle/fluid/io.py +++ b/python/paddle/fluid/io.py @@ -2154,7 +2154,6 @@ def set_var(var, ndarray): def load_program_state(model_path, var_list=None): """ - :api_attr: Static Graph Load program state from local file @@ -2169,6 +2168,7 @@ def load_program_state(model_path, var_list=None): state_dict(dict): the dict store Parameter and optimizer information Examples: + .. code-block:: python import paddle diff --git a/python/paddle/fluid/layers/nn.py b/python/paddle/fluid/layers/nn.py index 1fdf59948345b..200e8feec1e6a 100755 --- a/python/paddle/fluid/layers/nn.py +++ b/python/paddle/fluid/layers/nn.py @@ -6781,7 +6781,10 @@ def lod_append(x, level): x = fluid.layers.data(name='x', shape=[6, 10], lod_level=1) out = fluid.layers.lod_append(x, [1,1,1,1,1,1]) """ - from collections import Iterable + try: + from collections.abc import Iterable + except: + from collections import Iterable if x is None: raise ValueError("Input(x) can't be None.") if (not isinstance(level, Iterable)) and (not isinstance(level, Variable)): @@ -11850,8 +11853,7 @@ def _elementwise_op(helper): def scale(x, scale=1.0, bias=0.0, bias_after_scale=True, act=None, name=None): """ - Scale operator. - + Putting scale and bias to the input Tensor as following: ``bias_after_scale`` is True: @@ -11876,6 +11878,7 @@ def scale(x, scale=1.0, bias=0.0, bias_after_scale=True, act=None, name=None): Tensor: Output tensor of scale operator, with shape and data type same as input. Examples: + .. code-block:: python # scale as a float32 number diff --git a/python/paddle/fluid/layers/rnn.py b/python/paddle/fluid/layers/rnn.py index 1b9c87f1c0d06..707a1dc2cbc2f 100644 --- a/python/paddle/fluid/layers/rnn.py +++ b/python/paddle/fluid/layers/rnn.py @@ -33,6 +33,10 @@ from ..framework import _non_static_mode from ..param_attr import ParamAttr from ..data_feeder import check_variable_and_dtype, check_type, check_dtype +try: + from collections.abc import Sequence +except: + from collections import Sequence __all__ = [ 'RNNCell', @@ -163,7 +167,7 @@ def _is_shape_sequence(seq): # TODO: Add check for the illegal if isinstance(seq, dict): return True - return (isinstance(seq, collections.Sequence) and + return (isinstance(seq, Sequence) and not isinstance(seq, six.string_types)) class Shape(object): diff --git a/python/paddle/fluid/layers/tensor.py b/python/paddle/fluid/layers/tensor.py index 693fbf20e64a8..a9b1fa6ff0205 100644 --- a/python/paddle/fluid/layers/tensor.py +++ b/python/paddle/fluid/layers/tensor.py @@ -1470,6 +1470,11 @@ def range(start, end, step, dtype, name=None): # [3, 4, 5, 6] """ + out_shape = None + if not isinstance(start, Variable) and not isinstance( + end, Variable) and not isinstance(step, Variable): + out_shape = [int(math.ceil((end - start) / step))] + if not isinstance(dtype, core.VarDesc.VarType): dtype = convert_np_dtype_to_dtype_(dtype) @@ -1500,11 +1505,6 @@ def range(start, end, step, dtype, name=None): out.stop_gradient = True return out - out_shape = None - if not isinstance(start, Variable) and not isinstance( - end, Variable) and not isinstance(step, Variable): - out_shape = [int(math.ceil((end - start) / step))] - check_dtype(dtype, 'dtype', ['float32', 'float64', 'int32', 'int64'], 'range/arange') helper = LayerHelper('range', **locals()) @@ -1516,6 +1516,8 @@ def range(start, end, step, dtype, name=None): 'Step': step}, outputs={'Out': out}) out.stop_gradient = True + if out_shape is not None: + out.desc.set_shape(out_shape) return out diff --git a/python/paddle/fluid/layers/utils.py b/python/paddle/fluid/layers/utils.py index c30f41f6a20d9..5d781a437fe8f 100644 --- a/python/paddle/fluid/layers/utils.py +++ b/python/paddle/fluid/layers/utils.py @@ -21,6 +21,10 @@ from ..data_feeder import convert_dtype, check_variable_and_dtype, check_type, check_dtype from ..layer_helper import LayerHelper from sys import version_info +try: + from collections.abc import Sequence +except: + from collections import Sequence def convert_to_list(value, n, name, dtype=int): @@ -74,8 +78,7 @@ def is_sequence(seq): """ if isinstance(seq, dict): return True - return (isinstance(seq, collections.Sequence) and - not isinstance(seq, six.string_types)) + return (isinstance(seq, Sequence) and not isinstance(seq, six.string_types)) def _hash_with_id(*args): @@ -148,7 +151,7 @@ def _sequence_like(instance, args): return type(instance)((key, result[key]) for key in six.iterkeys(instance)) elif (isinstance(instance, tuple) and hasattr(instance, "_fields") and - isinstance(instance._fields, collections.Sequence) and + isinstance(instance._fields, Sequence) and all(isinstance(f, six.string_types) for f in instance._fields)): # This is a namedtuple return type(instance)(*args) diff --git a/python/paddle/fluid/param_attr.py b/python/paddle/fluid/param_attr.py index c3ee11ff5d906..a10ce1ce808f6 100644 --- a/python/paddle/fluid/param_attr.py +++ b/python/paddle/fluid/param_attr.py @@ -30,16 +30,17 @@ class ParamAttr(object): """ - Create a object to represent the attribute of parameter. The attributes are: - name, initializer, learning rate, regularizer, trainable, gradient clip, - and model average. - + Note: ``gradient_clip`` of ``ParamAttr`` HAS BEEN DEPRECATED since 2.0. Please use ``need_clip`` in ``ParamAttr`` to speficiy the clip scope. There are three clipping strategies: :ref:`api_paddle_nn_ClipGradByGlobalNorm` , :ref:`api_paddle_nn_ClipGradByNorm` , :ref:`api_paddle_nn_ClipGradByValue` . + Create a object to represent the attribute of parameter. The attributes are: + name, initializer, learning rate, regularizer, trainable, gradient clip, + and model average. + Parameters: name (str, optional): The parameter's name. Default None, meaning that the name would be created automatically. @@ -63,6 +64,7 @@ class ParamAttr(object): ParamAttr Object. Examples: + .. code-block:: python import paddle @@ -213,24 +215,22 @@ def _to_kwargs(self, with_initializer=False): class WeightNormParamAttr(ParamAttr): r""" - :api_attr: Static Graph Note: Please use 'paddle.nn.utils.weight_norm' in dygraph mode. - + + Note: + ``gradient_clip`` of ``ParamAttr`` HAS BEEN DEPRECATED since 2.0. + Please use ``need_clip`` in ``ParamAttr`` to speficiy the clip scope. + There are three clipping strategies: :ref:`api_paddle_nn_ClipGradByGlobalNorm` , + :ref:`api_paddle_nn_ClipGradByNorm` , :ref:`api_paddle_nn_ClipGradByValue` . + Parameter of weight Norm. Weight Norm is a reparameterization of the weight vectors in a neural network that decouples the magnitude of those weight vectors from their direction. Weight Norm has been implemented as discussed in this paper: `Weight Normalization: A Simple Reparameterization to Accelerate Training of Deep Neural Networks `_. - - Note: - ``gradient_clip`` of ``ParamAttr`` HAS BEEN DEPRECATED since 2.0. - Please use ``need_clip`` in ``ParamAttr`` to speficiy the clip scope. - There are three clipping strategies: :ref:`api_paddle_nn_ClipGradByGlobalNorm` , - :ref:`api_paddle_nn_ClipGradByNorm` , :ref:`api_paddle_nn_ClipGradByValue` . - Args: dim(int, optional): Dimension over which to compute the norm. Dim is a non-negative @@ -258,6 +258,7 @@ class WeightNormParamAttr(ParamAttr): need_clip (bool, optional): Whether the parameter gradient need to be cliped in optimizer. Default is True. Examples: + .. code-block:: python import paddle diff --git a/python/paddle/fluid/reader.py b/python/paddle/fluid/reader.py index 0f5f217442135..3ea3af9ed1cb5 100644 --- a/python/paddle/fluid/reader.py +++ b/python/paddle/fluid/reader.py @@ -18,11 +18,14 @@ import numpy as np import threading import paddle +import time +import copy + from .framework import Program, Variable, program_guard, default_main_program, default_startup_program, _non_static_mode, cpu_places, _current_expected_place, _in_eager_without_dygraph_check from .executor import global_scope from .data_feeder import DataFeeder, BatchedTensorProvider from .multiprocess_utils import multiprocess_queue_set, CleanupFuncRegistrar, _cleanup_mmap, _cleanup, _set_SIGCHLD_handler -from .dataloader import BatchSampler, Dataset, IterableDataset +from .dataloader import BatchSampler, Dataset, IterableDataset, Subset from .dataloader.dataloader_iter import _DataLoaderIterSingleProcess, _DataLoaderIterMultiProcess, _DatasetKind, default_collate_fn from .dataloader.batch_sampler import _InfiniteIterableSampler from .layers.io import monkey_patch_reader_methods, _copy_reader_var_, double_buffer @@ -36,10 +39,8 @@ import os import multiprocessing import signal - # NOTE: queue has a different name in python2 and python3 import queue - # NOTE: [ avoid hanging & failed quickly ] These value is used in getting data from another process QUEUE_GET_TIMEOUT = 60 @@ -49,6 +50,16 @@ KEEP_DATA_LOADER_ORDER = True USE_PINNED_MEMORY = None +# AutoTune Flags +USE_AUTOTUNE = False +TUNING_STEPS = 500 + + +def set_autotune_config(use_autotune, tuning_steps=500): + global USE_AUTOTUNE + USE_AUTOTUNE = use_autotune + global TUNING_STEPS + TUNING_STEPS = tuning_steps def keep_data_loader_order(*args): @@ -143,6 +154,122 @@ def _check_input_array(cls, item): return arr +class AuToTune(object): + def __init__(self, loader): + self.loader = loader + self.max_num_worker = multiprocessing.cpu_count() / 2 + + def __call__(self): + # use default loader + if (not USE_AUTOTUNE) or (not self.need_autotune()): + return self.loader.num_workers + + # get autotune loader + auto_tune_loader = self.get_autotune_loader() + if auto_tune_loader is None: + return self.loader.num_workers + + # pick the best num_workers + auto_tune_start = time.time() + logging.debug("========= DataLoader Auto Tune =========") + logging.debug("User config for DataLoader: " + str( + self.loader.num_workers)) + best_num_workers = 0 + min_cost = float("inf") + logging.debug("Tuning Range for num_workers: 0 ~ " + str( + self.max_num_worker)) + num_workers = 0 + while num_workers < self.max_num_worker: + auto_tune_loader.num_workers = num_workers + avg_cost = self.evaluate_reader_cost(auto_tune_loader) + if min_cost * 0.75 > avg_cost: + min_cost = avg_cost + best_num_workers = num_workers + else: + update_num = self.is_best(auto_tune_loader, best_num_workers, + min_cost, self.max_num_worker) + if update_num == best_num_workers: + break + else: + best_num_workers = update_num + logging.debug("num_workers: " + str(num_workers) + " avg_cost: " + + str(avg_cost)) + num_workers += 2 + logging.info("auto_tune dataLoader best_num_workers: " + str( + best_num_workers)) + logging.debug("AutoTuning Cost for DataLoader: " + str(time.time( + ) - auto_tune_start) + ' seconds') + + # tune the default loader's num_workers + return best_num_workers + + def need_autotune(self): + if (sys.platform == 'darwin' or sys.platform == 'win32'): + return False + else: + return True + + def get_sub_dataset(self, dataset, batch_size): + num_samples = min(batch_size * TUNING_STEPS, len(dataset)) + sub_dataset = Subset(dataset, indices=list(range(num_samples))) + return sub_dataset + + def get_autotune_loader(self): + loader = copy.copy(self.loader) + batch_size = self.loader.batch_sampler.batch_size + if isinstance(self.loader.batch_sampler, + paddle.io.DistributedBatchSampler): + dataset = self.loader.batch_sampler.dataset + sub_dataset = self.get_sub_dataset(dataset, batch_size) + loader.batch_sampler = paddle.io.DistributedBatchSampler( + dataset=sub_dataset, + batch_size=batch_size, + num_replicas=self.loader.batch_sampler.nranks, + rank=self.loader.batch_sampler.local_rank, + shuffle=self.loader.batch_sampler.shuffle, + drop_last=self.loader.batch_sampler.drop_last) + elif isinstance(self.loader.batch_sampler, paddle.io.BatchSampler): + dataset = self.loader.batch_sampler.sampler.data_source + sub_dataset = self.get_sub_dataset(dataset, batch_size) + loader.batch_sampler = paddle.io.BatchSampler( + dataset=sub_dataset, + batch_size=batch_size, + drop_last=self.loader.batch_sampler.drop_last) + else: + loader = None + return loader + + def evaluate_reader_cost(self, reader): + costs = [] + avg_cost = 0 + start = time.time() + for i, data in enumerate(reader): + costs.append(time.time() - start) + start = time.time() + if len(costs) > 2: + avg_cost = sum(costs[2:]) / len(costs[2:]) + else: + avg_cost = sum(costs[0:]) / len(costs[0:]) + return avg_cost + + def is_best(self, reader, best_workers, best_time, num_work_boundary): + step = 0 + num_workers = best_workers + 1 + boundary = 1 + while num_workers < num_work_boundary and step < 5: + self.loader.num_workers = num_workers + time = self.evaluate_reader_cost(reader) + logging.debug("for back num_workers: " + str(num_workers) + + " avg_cost: " + str(time)) + step += 1 + if (time < best_time * 0.70 * boundary): + return num_workers + else: + num_workers += 1 + boundary *= 0.80 + return best_workers + + class DataLoader(object): """ DataLoader prodives an iterator which iterates given dataset @@ -409,6 +536,7 @@ def __init__(self, self._persistent_workers = persistent_workers self._iterator = None + self.num_workers = AuToTune(self).__call__() def __len__(self): if self.dataset_kind == _DatasetKind.ITER: diff --git a/python/paddle/fluid/tests/custom_op/test_custom_tanh_double_grad.py b/python/paddle/fluid/tests/custom_op/test_custom_tanh_double_grad.py index 1127108c361ad..5664c00d74f89 100644 --- a/python/paddle/fluid/tests/custom_op/test_custom_tanh_double_grad.py +++ b/python/paddle/fluid/tests/custom_op/test_custom_tanh_double_grad.py @@ -21,7 +21,8 @@ from paddle.utils.cpp_extension import load, get_build_directory from paddle.utils.cpp_extension.extension_utils import run_cmd from utils import paddle_includes, extra_cc_args, extra_nvcc_args -from paddle.fluid.framework import _test_eager_guard +from paddle.fluid.framework import _test_eager_guard, _enable_legacy_dygraph +_enable_legacy_dygraph() # Because Windows don't use docker, the shared lib already exists in the # cache dir, it will not be compiled again unless the shared lib is removed. diff --git a/python/paddle/fluid/tests/unittests/CMakeLists.txt b/python/paddle/fluid/tests/unittests/CMakeLists.txt index 5235b7f1e88ab..15dd3d8b8f509 100755 --- a/python/paddle/fluid/tests/unittests/CMakeLists.txt +++ b/python/paddle/fluid/tests/unittests/CMakeLists.txt @@ -25,6 +25,7 @@ list(APPEND DIST_TEST_OPS test_ir_pass_pipeline) list(APPEND DIST_TEST_OPS test_static_model_parallel) list(APPEND DIST_TEST_OPS test_static_model_parallel_fused_feedforward) list(APPEND DIST_TEST_OPS test_static_model_parallel_fused_attention) +list(APPEND DIST_TEST_OPS test_static_model_parallel_fused_multi_transformer) list(APPEND DIST_TEST_OPS test_parallel_dygraph_se_resnext) list(APPEND DIST_TEST_OPS test_parallel_dygraph_sparse_embedding) list(APPEND DIST_TEST_OPS test_parallel_dygraph_sparse_embedding_over_height) @@ -128,19 +129,13 @@ if(NOT WITH_GPU) LIST(REMOVE_ITEM TEST_OPS test_fused_feedforward_op) LIST(REMOVE_ITEM TEST_OPS test_fused_attention_op) LIST(REMOVE_ITEM TEST_OPS test_fused_attention_op_api) + LIST(REMOVE_ITEM TEST_OPS test_fused_multi_transformer_op) LIST(REMOVE_ITEM TEST_OPS test_fused_transformer_encoder_layer) - LIST(REMOVE_ITEM TEST_OPS test_fused_gemm_epilogue_op) - LIST(REMOVE_ITEM TEST_OPS test_fused_gemm_epilogue_grad_op) - LIST(REMOVE_ITEM TEST_OPS test_fuse_gemm_epilogue_pass) endif() -if (WITH_GPU) - if (CUDA_VERSION LESS 11.6) - LIST(REMOVE_ITEM TEST_OPS test_fused_gemm_epilogue_op) - LIST(REMOVE_ITEM TEST_OPS test_fused_gemm_epilogue_grad_op) - LIST(REMOVE_ITEM TEST_OPS test_fuse_gemm_epilogue_pass) - endif() -endif() +LIST(REMOVE_ITEM TEST_OPS test_fused_gemm_epilogue_op) +LIST(REMOVE_ITEM TEST_OPS test_fused_gemm_epilogue_grad_op) +LIST(REMOVE_ITEM TEST_OPS test_fuse_gemm_epilogue_pass) if(((NOT WITH_ROCM) AND (NOT WITH_GPU)) OR WIN32) LIST(REMOVE_ITEM TEST_OPS test_c_comm_init_all_op) @@ -644,6 +639,15 @@ py_test_modules(test_imperative_static_runner_mnist MODULES test_imperative_stat FLAGS_cudnn_deterministic=1) py_test_modules(test_imperative_static_runner_while MODULES test_imperative_static_runner_while ENVS FLAGS_cudnn_deterministic=1) + +if ((WITH_GPU) AND (CUDA_VERSION GREATER_EQUAL 11.6)) + py_test_modules(test_fused_gemm_epilogue_op MODULES test_fused_gemm_epilogue_op) + py_test_modules(test_fused_gemm_epilogue_grad_op MODULES test_fused_gemm_epilogue_grad_op) + py_test_modules(test_fused_gemm_epilogue_op_with_es MODULES test_fused_gemm_epilogue_op ENVS FLAGS_cublaslt_exhaustive_search_times=30) + py_test_modules(test_fused_gemm_epilogue_grad_op_with_es MODULES test_fused_gemm_epilogue_grad_op ENVS FLAGS_cublaslt_exhaustive_search_times=30) + py_test_modules(test_fuse_gemm_epilogue_pass MODULES test_fuse_gemm_epilogue_pass) +endif() + set_tests_properties(test_conv2d_op PROPERTIES LABELS "RUN_TYPE=EXCLUSIVE") set_tests_properties(test_faster_tokenizer_op PROPERTIES LABELS "RUN_TYPE=EXCLUSIVE") set_tests_properties(test_conv2d_op_depthwise_conv PROPERTIES LABELS "RUN_TYPE=EXCLUSIVE") @@ -910,6 +914,7 @@ set_tests_properties(test_parallel_executor_crf test_sync_batch_norm_op test_inp test_parallel_executor_seresnext_with_fuse_all_reduce_gpu test_distributed_fused_lamb_op_with_clip test_distributed_fused_lamb_op_without_clip + test_distributed_fused_lamb_op_with_gradient_merge test_parallel_executor_fetch_isolated_var PROPERTIES LABELS "RUN_TYPE=DIST") @@ -1043,6 +1048,7 @@ set_tests_properties(test_row_conv_op PROPERTIES TIMEOUT 120) set_tests_properties(test_parallel_executor_seresnext_with_fuse_all_reduce_gpu PROPERTIES TIMEOUT 120) set_tests_properties(test_distributed_fused_lamb_op_with_clip PROPERTIES TIMEOUT 120) set_tests_properties(test_distributed_fused_lamb_op_without_clip PROPERTIES TIMEOUT 120) +set_tests_properties(test_distributed_fused_lamb_op_with_gradient_merge PROPERTIES TIMEOUT 120) set_tests_properties(test_elementwise_min_op PROPERTIES TIMEOUT 120) set_tests_properties(test_nan_inf PROPERTIES TIMEOUT 120) set_tests_properties(test_deformable_conv_v1_op PROPERTIES TIMEOUT 300) @@ -1063,6 +1069,7 @@ set_tests_properties(test_lstm_cudnn_op PROPERTIES TIMEOUT 120) set_tests_properties(test_stack_op PROPERTIES TIMEOUT 120) set_tests_properties(test_bilinear_interp_v2_op PROPERTIES TIMEOUT 120) set_tests_properties(test_svd_op PROPERTIES TIMEOUT 80) +set_tests_properties(test_einsum_op PROPERTIES TIMEOUT 120) set_tests_properties(test_qr_op PROPERTIES TIMEOUT 60) set_tests_properties(test_deformable_psroi_pooling PROPERTIES TIMEOUT 120) set_tests_properties(test_trilinear_interp_v2_op PROPERTIES TIMEOUT 120) @@ -1184,6 +1191,7 @@ if((WITH_ROCM OR WITH_GPU) AND NOT WIN32) set_tests_properties(test_static_model_parallel PROPERTIES TIMEOUT 240) set_tests_properties(test_static_model_parallel_fused_feedforward PROPERTIES TIMEOUT 120) set_tests_properties(test_static_model_parallel_fused_attention PROPERTIES TIMEOUT 120) + set_tests_properties(test_static_model_parallel_fused_multi_transformer PROPERTIES TIMEOUT 120) set_tests_properties(test_collective_split_embedding test_collective_split_embedding_none_divisible test_collective_split_row_linear diff --git a/python/paddle/fluid/tests/unittests/cc_imp_py_test.cc b/python/paddle/fluid/tests/unittests/cc_imp_py_test.cc index 8609aff1fa556..a0b9ec5f9f6d4 100644 --- a/python/paddle/fluid/tests/unittests/cc_imp_py_test.cc +++ b/python/paddle/fluid/tests/unittests/cc_imp_py_test.cc @@ -50,7 +50,8 @@ TEST(CC, IMPORT_PY) { // 3. C/C++ Run Python file std::string file_name(cwd); file_name.append("/test_install_check.py"); - FILE* fp = _Py_fopen(file_name.c_str(), "r+"); + PyObject* obj = Py_BuildValue("s", file_name.c_str()); + FILE* fp = _Py_fopen_obj(obj, "r+"); ASSERT_TRUE(fp != NULL); ASSERT_FALSE(PyRun_SimpleFile(fp, file_name.c_str())); diff --git a/python/paddle/fluid/tests/unittests/check_nan_inf_base_dygraph.py b/python/paddle/fluid/tests/unittests/check_nan_inf_base_dygraph.py index 08bab306df1b1..f4217d11f2d9b 100644 --- a/python/paddle/fluid/tests/unittests/check_nan_inf_base_dygraph.py +++ b/python/paddle/fluid/tests/unittests/check_nan_inf_base_dygraph.py @@ -25,6 +25,8 @@ import paddle import paddle.nn as nn +from paddle.fluid.framework import _enable_legacy_dygraph +_enable_legacy_dygraph() np.random.seed(0) diff --git a/python/paddle/fluid/tests/unittests/dist_fleet_ctr.py b/python/paddle/fluid/tests/unittests/dist_fleet_ctr.py index 2bd397b0ef3f5..be5118f0acc18 100644 --- a/python/paddle/fluid/tests/unittests/dist_fleet_ctr.py +++ b/python/paddle/fluid/tests/unittests/dist_fleet_ctr.py @@ -339,5 +339,9 @@ def do_dataset_training(self, fleet): if dirname: fleet.save_persistables(exe, dirname=dirname) + cache_dirname = os.getenv("SAVE_CACHE_DIRNAME", None) + if cache_dirname: + fleet.save_cache_model(cache_dirname) + if __name__ == "__main__": runtime_main(TestDistCTR2x2) diff --git a/python/paddle/fluid/tests/unittests/distributed_fused_lamb_test_base.py b/python/paddle/fluid/tests/unittests/distributed_fused_lamb_test_base.py index 00d2a1f71d6bd..0af7d40a2f02e 100644 --- a/python/paddle/fluid/tests/unittests/distributed_fused_lamb_test_base.py +++ b/python/paddle/fluid/tests/unittests/distributed_fused_lamb_test_base.py @@ -149,6 +149,7 @@ def run_model(use_distributed_lamb, use_fp16, use_master_param_norm, **kwargs): kwargs['exclude_from_weight_decay_fn'] = exclude_fn kwargs['lamb_weight_decay'] = 0.1 + gm_steps = kwargs['gradient_accumulation_steps'] if use_distributed_lamb: optimizer_class = DistributedFusedLamb kwargs = dict(kwargs) @@ -163,6 +164,7 @@ def run_model(use_distributed_lamb, use_fp16, use_master_param_norm, **kwargs): ) kwargs['grad_clip'] = GradClipDecorator(base_clip, clip_after_allreduce) + kwargs.pop('gradient_accumulation_steps', None) optimizer = optimizer_class(**kwargs) get_parameter = optimizer._get_parameter @@ -173,6 +175,7 @@ def run_model(use_distributed_lamb, use_fp16, use_master_param_norm, **kwargs): if use_fp16: if not use_distributed_lamb: optimizer._multi_precision = True + optimizer = paddle.static.amp.decorate( optimizer, amp_list, @@ -180,6 +183,13 @@ def run_model(use_distributed_lamb, use_fp16, use_master_param_norm, **kwargs): use_dynamic_loss_scaling=False, use_pure_fp16=use_fp16, use_fp16_guard=use_fp16) + amp_init = optimizer.amp_init + else: + amp_init = None + + if gm_steps > 1 and not use_distributed_lamb: + optimizer = paddle.fluid.optimizer.GradientMergeOptimizer( + optimizer, k_steps=gm_steps, avg=False) params_grads = optimizer.backward(loss, startup) op_num = len(main.global_block().ops) @@ -211,7 +221,7 @@ def gen_random_grad_tensor(grad): return grad_t def reader(): - for _ in range(5): + for _ in range(6): yield dict( [(grad.name, gen_random_grad_tensor(grad)) for grad in grads]) @@ -223,8 +233,8 @@ def reader(): place = paddle.CUDAPlace(dev_id) exe = paddle.static.Executor(place) exe.run(startup) - if use_fp16: - optimizer.amp_init(place) + if amp_init is not None: + amp_init(place) master_p_ts = [] for p in params: @@ -258,10 +268,12 @@ def config(self): distutils.util.strtobool( os.getenv('CLIP_AFTER_ALLREDUCE', 'True'))) max_global_norm = float(os.getenv('MAX_GLOBAL_NORM', -1.0)) + gm_steps = int(os.getenv('GRADIENT_MERGE_STEPS', 1)) print('clip_after_allreduce = {}, max_global_norm = {}'.format( clip_after_allreduce, max_global_norm)) return { 'clip_after_allreduce': clip_after_allreduce, + 'gradient_accumulation_steps': gm_steps, 'grad_clip': paddle.nn.ClipGradByGlobalNorm(max_global_norm) if max_global_norm > 0 else None, } diff --git a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_simnet_v2.py b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_simnet_v2.py index 872d419ff8928..ab836b088b09f 100644 --- a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_simnet_v2.py +++ b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_simnet_v2.py @@ -20,6 +20,9 @@ from simnet_dygraph_model_v2 import BOW, HingeLoss +from paddle.fluid.framework import _enable_legacy_dygraph +_enable_legacy_dygraph() + SEED = 102 random.seed(SEED) diff --git a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_spec_names.py b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_spec_names.py index 361fcbf9c73f5..bafc4707c4ad9 100644 --- a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_spec_names.py +++ b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_spec_names.py @@ -16,6 +16,8 @@ from paddle.nn import Layer import numpy as np import unittest +from paddle.fluid.framework import _enable_legacy_dygraph +_enable_legacy_dygraph() class Net(Layer): diff --git a/python/paddle/fluid/tests/unittests/gradient_checker.py b/python/paddle/fluid/tests/unittests/gradient_checker.py index 562d52668ce5b..32a7e442ea961 100644 --- a/python/paddle/fluid/tests/unittests/gradient_checker.py +++ b/python/paddle/fluid/tests/unittests/gradient_checker.py @@ -27,6 +27,10 @@ from paddle.fluid.executor import Executor from paddle.fluid.backward import _append_grad_suffix_, _as_list from paddle.fluid.framework import _test_eager_guard +try: + from collections.abc import Sequence +except: + from collections import Sequence def _product(t): @@ -60,19 +64,6 @@ def _get_item(t, i, np_dtype): raise ValueError("Not supported data type " + str(np_dtype)) -def _get_item_for_dygraph(t, i, np_dtype): - if np_dtype == np.float16: - np_t = t.numpy().astype(np.float16) - elif np_dtype == np.float32: - np_t = t.numpy().astype(np.float32) - elif np_dtype == np.float64: - np_t = t.numpy().astype(np.float64) - else: - raise ValueError("Not supported data type " + str(np_dtype)) - np_t = np_t.flatten() - return np_t[i] - - def _set_item(t, i, e, np_dtype): if np_dtype == np.float16: np_t = np.array(t).astype(np.float16) @@ -89,22 +80,6 @@ def _set_item(t, i, e, np_dtype): raise ValueError("Not supported data type " + str(np_dtype)) -def _set_item_for_dygraph(t, i, e, np_dtype): - if np_dtype == np.float16: - np_t = t.numpy().astype(np.float16) - elif np_dtype == np.float32: - np_t = t.numpy().astype(np.float32) - elif np_dtype == np.float64: - np_t = t.numpy().astype(np.float64) - else: - raise ValueError("Not supported data type " + str(np_dtype)) - shape = np_t.shape - np_t = np_t.flatten() - np_t[i] = e - np_t = np_t.reshape(shape) - paddle.assign(np_t, t) - - def set_var_in_scope(scope, place, name, value, recursive_seq_len=None): t = scope.var(name).get_tensor() t.set(value, place) @@ -120,7 +95,7 @@ def var_to_np_array_in_scope(scope, place, name): def make_jacobian(x, y_size, np_dtype): if isinstance(x, fluid.framework.Variable): return np.zeros((_product(x.shape), y_size), dtype=np_dtype) - elif isinstance(x, collections.Sequence): + elif isinstance(x, Sequence): jacobians = list( filter(lambda t: t is not None, (make_jacobian( item, y_size, np_dtype) for item in x))) @@ -169,8 +144,6 @@ def run(): np_type = dtype_to_np_dtype(x.dtype) jacobian = [make_jacobian(x, _product(yi.shape), np_type) for yi in y] - if np_type == np.float64: - delta = 1e-5 for i in six.moves.xrange(x_size): orig = _get_item(x_t, i, np_type) x_pos = orig + delta @@ -545,7 +518,12 @@ def triple_grad_check(x, rtol=rtol) -def get_static_double_grad(x, y, x_init=None, dy_init=None, place=None): +def get_static_double_grad(x, + y, + x_init=None, + dy_init=None, + place=None, + program=None): """ Get Double Grad result of static graph. @@ -555,11 +533,14 @@ def get_static_double_grad(x, y, x_init=None, dy_init=None, place=None): x_init (numpy.array|list[numpy.array]|None): the init value for input x. dy_init (numpy.array|list[numpy.array]|None): the init value for output y. place (fluid.CPUPlace or fluid.CUDAPlace): the device. + program (Program|None): a Program with forward pass. + If None, use fluid.default_main_program(). Returns: A list of numpy array that stores second derivative result calulated by static graph. """ - program = fluid.default_main_program() + if program is None: + program = fluid.default_main_program() scope = fluid.executor.global_scope() y_grads = [] for i in six.moves.xrange(len(y)): @@ -635,7 +616,10 @@ def get_static_double_grad(x, y, x_init=None, dy_init=None, place=None): return ddx_res -def get_eager_double_grad(func, x_init=None, dy_init=None): +def get_eager_double_grad(func, + x_init=None, + dy_init=None, + return_mid_result=False): """ Get Double Grad result of dygraph. @@ -643,8 +627,13 @@ def get_eager_double_grad(func, x_init=None, dy_init=None): func: A wrapped dygraph function that its logic is equal to static program x_init (numpy.array|list[numpy.array]|None): the init value for input x. dy_init (numpy.array|list[numpy.array]|None): the init value for gradient of output. + return_mid_result (bool): A flag that controls the return content. Returns: - A list of numpy array that stores second derivative result calulated by dygraph + If 'return_mid_result' set True. + the second order derivative and the inputs of second order derivative's calculation + will be returned for higher order derivative's calculation. + If 'return_mid_result' set False. + A list of numpy array that stores second derivative result calulated by dygraph. """ inputs = [] dys = [] @@ -664,13 +653,25 @@ def get_eager_double_grad(func, x_init=None, dy_init=None): # calcluate second derivative inputs = inputs + dys ddys = [] + if return_mid_result: + create_graph = True + else: + create_graph = False + for d_input in d_inputs: d_input.stop_gradient = False ddy = paddle.ones(shape=d_input.shape, dtype=d_input.dtype) ddy.stop_gradient = False ddys.append(ddy) - dd_inputs = paddle.grad(outputs=d_inputs, inputs=inputs, grad_outputs=ddys) - return [dd_input.numpy() for dd_input in dd_inputs] + dd_inputs = paddle.grad( + outputs=d_inputs, + inputs=inputs, + grad_outputs=ddys, + create_graph=create_graph) + if return_mid_result: + return dd_inputs, inputs + ddys + else: + return [dd_input.numpy() for dd_input in dd_inputs] def double_grad_check_for_dygraph(func, @@ -682,8 +683,9 @@ def double_grad_check_for_dygraph(func, rtol=1e-3, raise_exception=True): """ - Check gradients of gradients. This function will append backward to the - program before second order gradient check. + Check second order gradients of dygraph. This function will compare the + second order gradients of dygraph and second order gradients of static graph + to validate dygraph's correctness Args: func: A wrapped dygraph function that its logic is equal to static program @@ -734,3 +736,149 @@ def fail_test(msg): 'static:%s\n eager:%s\n' \ % (static_double_grad[i].name, eager_double_grad[i].name, str(place), static_double_grad[i], eager_double_grad[i]) return fail_test(msg) + + +def get_static_triple_grad(x, + y, + x_init=None, + dy_init=None, + place=None, + program=None): + """ + Get Triple Grad result of static graph. + + Args: + x (Variable|list[Variable]): input variables to the program. + y (Variable|list[Variable]): output variables to the program. + x_init (numpy.array|list[numpy.array]|None): the init value for input x. + dy_init (numpy.array|list[numpy.array]|None): the init value for output y. + place (fluid.CPUPlace or fluid.CUDAPlace): the device. + program (Program|None): a Program with forward pass. + If None, use fluid.default_main_program(). + Returns: + A list of numpy array that stores third derivative result calulated by static graph. + """ + if program is None: + program = fluid.default_main_program() + scope = fluid.executor.global_scope() + y_grads = [] + for i in six.moves.xrange(len(y)): + yi = y[i] + dyi_name = _append_grad_suffix_(yi.name) + np_type = dtype_to_np_dtype(yi.dtype) + dy = program.global_block().create_var( + name=dyi_name, shape=yi.shape, dtype=np_type, persistable=True) + dy.stop_gradient = False + set_var_in_scope(scope, place, dyi_name, dy_init[i]) + y_grads.append(dy) + + # append first order grads + dx = fluid.gradients(y, x, y_grads) + + # y_grads are the input of first-order backward, + # so, they are also the input of second-order backward. + x += y_grads + x_init += dy_init + y = dx + + x_grads_grads_init = [] + for dxi in dx: + np_type = dtype_to_np_dtype(dxi.dtype) + value = np.ones(dxi.shape, dtype=np_type) + x_grads_grads_init.append(value) + + return get_static_double_grad( + x, y, x_init, dy_init=x_grads_grads_init, place=place, program=program) + + +def get_eager_triple_grad(func, + x_init=None, + dy_init=None, + return_mid_result=False): + """ + Get triple Grad result of dygraph. + + Args: + func: A wrapped dygraph function that its logic is equal to static program + x_init (numpy.array|list[numpy.array]|None): the init value for input x. + dy_init (numpy.array|list[numpy.array]|None): the init value for gradient of output. + return_mid_result (list[Tensor], list[Tensor]): If set True, the + Returns: + A list of numpy array that stores second derivative result calulated by dygraph + """ + dd_y, dd_x = get_eager_double_grad( + func, x_init, dy_init, return_mid_result=True) + + # calcluate third derivative + dddys = [] + for dd_yi in dd_y: + dd_yi.stop_gradient = False + dddy = paddle.ones(shape=dd_yi.shape, dtype=dd_yi.dtype) + dddy.stop_gradient = False + dddys.append(dddy) + ddd_inputs = paddle.grad(outputs=dd_y, inputs=dd_x, grad_outputs=dddys) + return [ddd_input.numpy() for ddd_input in ddd_inputs] + + +def triple_grad_check_for_dygraph(func, + x, + y, + x_init=None, + place=None, + atol=1e-5, + rtol=1e-3, + raise_exception=True): + """ + Check third order gradients of dygraph. This function will compare the + third order gradients of dygraph and third order gradients of static graph + to validate dygraph's correctness + + Args: + func: A wrapped dygraph function that its logic is equal to static program + x (Variable|list[Variable]): input variables to the program. + y (Variable|list[Variable]): output variables to the program. + x_init (numpy.array|list[numpy.array]|None): the init value for input x. + place (fluid.CPUPlace or fluid.CUDAPlace): the device. + eps (float): perturbation for finite differences. + atol (float): absolute tolerance. + rtol (float): relative tolerance. + raise_exception (bool): whether to raise an exception if + the check fails. Default is True. + """ + + def fail_test(msg): + if raise_exception: + raise RuntimeError(msg) + return False + + # check input arguments + x = _as_list(x) + for v in x: + v.stop_gradient = False + v.persistable = True + y = _as_list(y) + + y_grads_init = [] + for yi in y: + np_type = dtype_to_np_dtype(yi.dtype) + v = np.random.random(size=yi.shape).astype(np_type) + y_grads_init.append(v) + + x_init = _as_list(x_init) + + paddle.disable_static() + with _test_eager_guard(): + eager_triple_grad = get_eager_triple_grad(func, x_init, y_grads_init) + paddle.enable_static() + + static_triple_grad = get_static_triple_grad(x, y, x_init, y_grads_init, + place) + + for i in six.moves.xrange(len(static_triple_grad)): + if not np.allclose(static_triple_grad[i], eager_triple_grad[i], rtol, + atol): + msg = 'Check eager double result fail. Mismatch between static_graph double grad %s ' \ + 'and eager double grad %s on %s,\n' \ + 'static:%s\n eager:%s\n' \ + % (static_triple_grad[i].name, eager_triple_grad[i].name, str(place), static_triple_grad[i], eager_triple_grad[i]) + return fail_test(msg) diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_shuffle_channel_detect_pass.py b/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_shuffle_channel_detect_pass.py new file mode 100644 index 0000000000000..828e92dc03426 --- /dev/null +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_shuffle_channel_detect_pass.py @@ -0,0 +1,138 @@ +# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from auto_scan_test import PassAutoScanTest, SkipReasons +from program_config import TensorConfig, ProgramConfig +import numpy as np +from functools import partial +import unittest + +from hypothesis import given, settings, seed, example, assume +import hypothesis.strategies as st + + +def product(input): + result = 1 + + for value in input: + result = result * value + + return result + + +class TestShuffleChannelMKLDNNDetectPass(PassAutoScanTest): + def is_program_valid(self, program_config: ProgramConfig) -> bool: + input_shape = program_config.inputs['input_data'].shape + first_reshape2_shape = program_config.ops[0].attrs['shape'] + transpose2_axis = program_config.ops[1].attrs['axis'] + second_reshape2_shape = program_config.ops[2].attrs['shape'] + + shape_prod = product(input_shape) + img_h = input_shape[-2] + img_w = input_shape[-1] + + if shape_prod != product(first_reshape2_shape) or shape_prod != product( + second_reshape2_shape): + return False + if len(input_shape) != 4 or len(first_reshape2_shape) != 5 or len( + second_reshape2_shape) != 4: + return False + if transpose2_axis != [0, 2, 1, 3, 4]: + return False + if first_reshape2_shape[-1] != img_w or first_reshape2_shape[ + -2] != img_h: + return False + if second_reshape2_shape[-1] != img_w or second_reshape2_shape[ + -2] != img_h: + return False + + return True + + def sample_program_config(self, draw): + input_shape = draw(st.sampled_from([[128, 32, 32]])) + first_reshape2_shape = draw( + st.sampled_from([[2, 64, 32, 32], [8, 16, 32, 32]])) + transpose2_axis = draw(st.sampled_from([[0, 2, 1, 3, 4], [0, 2, 1, 3]])) + second_reshape2_shape = draw( + st.sampled_from([[128, 32, 32], [128, 31, 32]])) + batch_size = draw(st.integers(min_value=1, max_value=10)) + + input_shape.insert(0, batch_size) + first_reshape2_shape.insert(0, batch_size) + second_reshape2_shape.insert(0, batch_size) + + def generate_input(): + return np.random.random(input_shape).astype(np.float32) + + ops_config = [{ + "op_type": "reshape2", + "op_inputs": { + "X": ["input_data"] + }, + "op_outputs": { + "Out": ["first_reshape2_output"], + "XShape": ["first_reshape2_xshape"] + }, + "op_attrs": { + 'shape': first_reshape2_shape + }, + }, { + "op_type": "transpose2", + "op_inputs": { + "X": ["first_reshape2_output"] + }, + "op_outputs": { + "Out": ["transpose2_output"], + "XShape": ["transpose2_xshape"] + }, + "op_attrs": { + 'axis': transpose2_axis + }, + }, { + "op_type": "reshape2", + "op_inputs": { + "X": ["transpose2_output"], + }, + "op_outputs": { + "Out": ["output_data"], + "XShape": ["second_reshape2_xshape"] + }, + "op_attrs": { + 'shape': second_reshape2_shape + } + }] + + ops = self.generate_op_config(ops_config) + + program_config = ProgramConfig( + ops=ops, + weights={}, + inputs={ + "input_data": TensorConfig(data_gen=partial(generate_input)) + }, + outputs=["output_data"]) + + return program_config + + def sample_predictor_configs(self, program_config): + config = self.create_inference_config(use_mkldnn=True) + yield config, ["shuffle_channel"], (1e-5, 1e-5) + + def test(self): + self.run_and_statis( + quant=False, passes=["shuffle_channel_mkldnn_detect_pass"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/python/paddle/fluid/tests/unittests/mlu/test_dropout_op_mlu.py b/python/paddle/fluid/tests/unittests/mlu/test_dropout_op_mlu.py new file mode 100644 index 0000000000000..f8984f5c6dfa4 --- /dev/null +++ b/python/paddle/fluid/tests/unittests/mlu/test_dropout_op_mlu.py @@ -0,0 +1,273 @@ +# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import print_function + +import numpy as np +import unittest +import sys +sys.path.append("..") +from op_test import OpTest, skip_check_grad_ci +import paddle +import paddle.fluid as fluid + +paddle.enable_static() + +SEED = 2022 + + +class TestDropoutOp(OpTest): + def setUp(self): + self.op_type = "dropout" + self.set_mlu() + self.init_dtype() + self.inputs = {'X': np.random.random((32, 64)).astype(self.dtype)} + self.attrs = { + 'dropout_prob': 0.0, + 'fix_seed': True, + 'is_test': False, + 'dropout_implementation': 'upscale_in_train' + } + self.outputs = { + 'Out': self.inputs['X'], + 'Mask': np.ones((32, 64)).astype('uint8') + } + + def init_dtype(self): + self.dtype = np.float32 + + def set_mlu(self): + self.__class__.use_mlu = True + self.place = paddle.device.MLUPlace(0) + + def test_check_output(self): + self.check_output_with_place(self.place) + + def test_check_grad_normal(self): + self.check_grad_with_place(self.place, ['X'], 'Out') + + +class TestDropoutOpInput1d(TestDropoutOp): + # change input shape + def setUp(self): + self.op_type = "dropout" + self.set_mlu() + self.init_dtype() + self.inputs = {'X': np.random.random((3, 62)).astype(self.dtype)} + self.attrs = { + 'dropout_prob': 0.0, + 'fix_seed': True, + 'is_test': False, + 'dropout_implementation': 'upscale_in_train' + } + self.outputs = { + 'Out': self.inputs['X'], + 'Mask': np.ones((3, 62)).astype('uint8') + } + + +class TestDropoutOpInput1d_1(TestDropoutOp): + # the input is 1-D + def setUp(self): + self.op_type = "dropout" + self.set_mlu() + self.init_dtype() + self.inputs = {'X': np.random.random((2000)).astype(self.dtype)} + self.attrs = { + 'dropout_prob': 0.0, + 'fix_seed': True, + 'is_test': False, + 'dropout_implementation': 'upscale_in_train' + } + self.outputs = { + 'Out': self.inputs['X'], + 'Mask': np.ones((2000)).astype('uint8') + } + + +class TestDropoutOp2(TestDropoutOp): + # the dropout_prob is 1.0 + def setUp(self): + self.op_type = "dropout" + self.set_mlu() + self.init_dtype() + self.inputs = {'X': np.random.random((32, 64)).astype(self.dtype)} + self.attrs = { + 'dropout_prob': 1.0, + 'fix_seed': True, + 'is_test': False, + 'dropout_implementation': 'upscale_in_train' + } + self.outputs = { + 'Out': np.zeros((32, 64)).astype('float32'), + 'Mask': np.zeros((32, 64)).astype('uint8') + } + + +class TestDropoutOp3(TestDropoutOp): + # the input dim is 3 + def setUp(self): + self.op_type = "dropout" + self.set_mlu() + self.init_dtype() + self.inputs = {'X': np.random.random((32, 64, 2)).astype(self.dtype)} + self.attrs = { + 'dropout_prob': 0.0, + 'fix_seed': True, + 'is_test': False, + 'dropout_implementation': 'upscale_in_train' + } + self.outputs = { + 'Out': self.inputs['X'], + 'Mask': np.ones((32, 64, 2)).astype('uint8') + } + + +@skip_check_grad_ci(reason="For inference, check_grad is not required.") +class TestDropoutOpInference(OpTest): + # is_test = True + def setUp(self): + self.op_type = "dropout" + self.set_mlu() + self.init_dtype() + self.inputs = {'X': np.random.random((32, 64)).astype(self.dtype)} + self.attrs = { + 'dropout_prob': 0.35, + 'fix_seed': True, + 'is_test': True, + 'dropout_implementation': 'upscale_in_train' + } + self.outputs = {'Out': self.inputs['X']} + + def init_dtype(self): + self.dtype = np.float32 + + def set_mlu(self): + self.__class__.use_mlu = True + self.place = paddle.device.MLUPlace(0) + + def test_check_output(self): + self.check_output_with_place(self.place) + + +@skip_check_grad_ci(reason="For inference, check_grad is not required.") +class TestDropoutOpInference2(TestDropoutOpInference): + def setUp(self): + self.op_type = "dropout" + self.set_mlu() + self.init_dtype() + self.inputs = {'X': np.random.random((32, 64, 3)).astype(self.dtype)} + self.attrs = { + 'dropout_prob': 0.75, + 'is_test': True, + 'dropout_implementation': 'upscale_in_train' + } + self.outputs = {'Out': self.inputs['X']} + + +class TestDropoutOpWithSeed(TestDropoutOp): + # the seed is a Tensor + def setUp(self): + self.op_type = "dropout" + self.set_mlu() + self.init_dtype() + self.inputs = { + "X": np.random.random((32, 64)).astype(self.dtype), + "Seed": np.asarray( + [125], dtype="int32") + } + self.attrs = { + 'dropout_prob': 0.0, + 'is_test': False, + 'dropout_implementation': 'upscale_in_train' + } + self.outputs = { + 'Out': self.inputs['X'], + 'Mask': np.ones((32, 64)).astype('uint8') + } + + +class TestDropoutOpFp16(TestDropoutOp): + # float16 + def init_dtype(self): + self.dtype = np.float16 + + def set_mlu(self): + self.__class__.use_mlu = True + self.place = paddle.device.MLUPlace(0) + self.__class__.no_need_check_grad = True + + +class TestDropoutAPI(unittest.TestCase): + def setUp(self): + np.random.seed(123) + self.places = [fluid.CPUPlace(), paddle.device.MLUPlace(0)] + + def check_static_result(self, place): + with fluid.program_guard(fluid.Program(), fluid.Program()): + input = fluid.data(name="input", shape=[40, 40], dtype="float32") + res1 = paddle.nn.functional.dropout( + x=input, p=0., training=False, mode='upscale_in_train') + res2 = paddle.nn.functional.dropout( + x=input, p=0., axis=0, training=True, mode='upscale_in_train') + res3 = paddle.nn.functional.dropout( + x=input, p=0., axis=0, training=False, mode='upscale_in_train') + res4 = paddle.nn.functional.dropout( + x=input, + p=0., + axis=[0, 1], + training=True, + mode='upscale_in_train') + res5 = paddle.nn.functional.dropout( + x=input, + p=0., + axis=[0, 1], + training=False, + mode='upscale_in_train') + res6 = paddle.nn.functional.dropout( + x=input, p=1., training=True, mode='upscale_in_train') + res7 = paddle.fluid.layers.dropout( + x=input, + dropout_prob=0., + dropout_implementation='upscale_in_train') + res8 = paddle.nn.functional.dropout( + x=input, + p=0., + axis=(0, 1), + training=False, + mode='upscale_in_train') + + in_np = np.random.random([40, 40]).astype("float32") + res_np = in_np + res_np2 = np.zeros_like(in_np) + + exe = fluid.Executor(place) + res_list = [res1, res2, res3, res4, res5, res7, res8] + for res in res_list: + fetches = exe.run(fluid.default_main_program(), + feed={"input": in_np}, + fetch_list=[res]) + self.assertTrue(np.allclose(fetches[0], res_np)) + fetches2 = exe.run(fluid.default_main_program(), + feed={"input": in_np}, + fetch_list=[res6]) + self.assertTrue(np.allclose(fetches2[0], res_np2)) + + def test_static(self): + for place in self.places: + self.check_static_result(place=place) + + +if __name__ == '__main__': + unittest.main() diff --git a/python/paddle/fluid/tests/unittests/op_test.py b/python/paddle/fluid/tests/unittests/op_test.py index cfe0d4e32ef7a..13c72bedefa8e 100644 --- a/python/paddle/fluid/tests/unittests/op_test.py +++ b/python/paddle/fluid/tests/unittests/op_test.py @@ -15,6 +15,7 @@ from __future__ import print_function import os +import sys import unittest import warnings import numpy as np @@ -37,20 +38,22 @@ from paddle.fluid.op import Operator from paddle.fluid.executor import Executor from paddle.fluid.framework import Program, OpProtoHolder, Variable, _current_expected_place -from paddle.fluid.tests.unittests.testsuite import ( +from paddle.fluid import unique_name +from paddle.fluid.dygraph.dygraph_to_static.utils import parse_arg_and_kwargs + +sys.path.append(os.path.abspath(os.path.dirname(__file__))) +from testsuite import ( create_op, set_input, append_input_output, append_loss_ops, ) -from paddle.fluid import unique_name -from paddle.fluid.tests.unittests.white_list import ( +from white_list import ( op_accuracy_white_list, check_shape_white_list, compile_vs_runtime_white_list, no_check_set_white_list, op_threshold_white_list, no_grad_set_white_list, ) -from paddle.fluid.dygraph.dygraph_to_static.utils import parse_arg_and_kwargs # For switch new eager mode globally g_is_in_eager = _in_eager_without_dygraph_check() @@ -341,6 +344,10 @@ def is_npu_op_test(): def is_mlu_op_test(): return hasattr(cls, "use_mlu") and cls.use_mlu == True + def is_custom_device_op_test(): + return hasattr( + cls, "use_custom_device") and cls.use_custom_device == True + if not hasattr(cls, "op_type"): raise AssertionError( "This test do not have op_type in class attrs, " @@ -364,7 +371,8 @@ def is_mlu_op_test(): and not is_mkldnn_op_test() \ and not is_rocm_op_test() \ and not is_npu_op_test() \ - and not is_mlu_op_test(): + and not is_mlu_op_test() \ + and not is_custom_device_op_test(): raise AssertionError( "This test of %s op needs check_grad with fp64 precision." % cls.op_type) @@ -1506,6 +1514,12 @@ def find_actual_value(self, name): return imperative_actual, imperative_actual_t def convert_uint16_to_float_ifneed(self, actual_np, expect_np): + if actual_np.dtype == np.uint16 and expect_np.dtype in [ + np.float32, np.float64 + ]: + self.rtol = 1.e-2 + else: + self.rtol = 1.e-5 if self.op_test.is_bfloat16_op(): if actual_np.dtype == np.uint16: actual_np = convert_uint16_to_float(actual_np) diff --git a/python/paddle/fluid/tests/unittests/process_group_nccl.py b/python/paddle/fluid/tests/unittests/process_group_nccl.py index 7aa83ad907914..3667633d3b38d 100644 --- a/python/paddle/fluid/tests/unittests/process_group_nccl.py +++ b/python/paddle/fluid/tests/unittests/process_group_nccl.py @@ -122,6 +122,29 @@ def test_create_process_group_nccl(self): print("test allreduce min api ok") + # test allreduce prod + # rank 0 + x = np.random.random(self.shape).astype(self.dtype) + tensor_x = paddle.to_tensor(x) + # rank 1 + y = np.random.random(self.shape).astype(self.dtype) + tensor_y = paddle.to_tensor(y) + + prod_result = np.multiply(x, y) + + if pg.rank() == 0: + task = dist.all_reduce( + tensor_x, dist.ReduceOp.PROD, use_calc_stream=False) + task.wait() + assert np.array_equal(tensor_x, prod_result) + else: + task = dist.all_reduce( + tensor_y, dist.ReduceOp.PROD, use_calc_stream=False) + task.wait() + assert np.array_equal(tensor_y, prod_result) + + print("test allreduce prod api ok") + # test broadcast # rank 0 x = np.random.random(self.shape).astype(self.dtype) @@ -332,6 +355,27 @@ def test_create_process_group_nccl(self): print("test reduce min api ok") + # test reduce product + # rank 0 + x = np.random.random(self.shape).astype(self.dtype) + tensor_x = paddle.to_tensor(x) + # rank 1 + y = np.random.random(self.shape).astype(self.dtype) + tensor_y = paddle.to_tensor(y) + + prod_result = np.multiply(x, y) + + if pg.rank() == 0: + task = dist.reduce( + tensor_x, 0, dist.ReduceOp.PROD, use_calc_stream=False) + task.wait() + assert np.array_equal(tensor_x, prod_result) + else: + task = dist.reduce( + tensor_y, 0, dist.ReduceOp.PROD, use_calc_stream=False) + task.wait() + + print("test reduce prod api ok") # test Scatter # rank 0 in_shape = list(self.shape) diff --git a/python/paddle/fluid/tests/unittests/static_model_parallel_fused_multi_transformer.py b/python/paddle/fluid/tests/unittests/static_model_parallel_fused_multi_transformer.py new file mode 100644 index 0000000000000..f9c5d4d78c866 --- /dev/null +++ b/python/paddle/fluid/tests/unittests/static_model_parallel_fused_multi_transformer.py @@ -0,0 +1,193 @@ +# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import print_function + +import numpy as np + +import paddle +import paddle.fluid as fluid +from test_dist_base import TestDistRunnerBase, runtime_main +from paddle.incubate.nn import FusedMultiTransformer +import paddle.distributed.fleet as fleet + +from paddle.fluid.data_feeder import check_variable_and_dtype, check_dtype +from paddle.fluid.dygraph.layers import Layer +from paddle.fluid.layer_helper import LayerHelper +from paddle.fluid import core +from paddle.nn.initializer import Constant + +paddle.enable_static() + + +def get_param_attr(weight, bias): + weight_attr = paddle.ParamAttr( + initializer=fluid.initializer.NumpyArrayInitializer(weight)) + bias_attr = paddle.ParamAttr( + initializer=fluid.initializer.NumpyArrayInitializer(bias)) + return weight_attr, bias_attr + + +DTYPE = "float32" +MODEL_PARALLEL_SIZE = 2 +num_head = 2 * MODEL_PARALLEL_SIZE +dim_head = 4 +hidden = num_head * dim_head +dim_ffn = 4 * hidden + + +def create_model(data, rank): + np.random.seed(2021) + ln_w = np.random.uniform(-1, 1, size=(hidden, )).astype(DTYPE) + ln_b = np.random.uniform(-1, 1, size=(hidden, )).astype(DTYPE) + qkv_w = np.random.uniform( + -1, 1, size=(3, num_head, dim_head, hidden)).astype(DTYPE) + qkv_b = np.random.uniform(-1, 1, size=(3, num_head, dim_head)).astype(DTYPE) + linear_w = np.random.uniform( + -1, 1, size=(num_head * dim_head, hidden)).astype(DTYPE) + linear_b = np.random.uniform(-1, 1, size=(hidden, )).astype(DTYPE) + + ffn_ln_w = np.random.uniform(-1, 1, size=(hidden, )).astype(DTYPE) + ffn_ln_b = np.random.uniform(-1, 1, size=(hidden, )).astype(DTYPE) + ffn1_w = np.random.uniform(-1, 1, size=(hidden, dim_ffn)).astype(DTYPE) + ffn1_b = np.random.uniform(-1, 1, size=(dim_ffn, )).astype(DTYPE) + ffn2_w = np.random.uniform(-1, 1, size=(dim_ffn, hidden)).astype(DTYPE) + ffn2_b = np.random.uniform(-1, 1, size=(hidden, )).astype(DTYPE) + + if rank is not None: + start = 0 if rank == 0 else (num_head // MODEL_PARALLEL_SIZE) + end = start + (num_head // MODEL_PARALLEL_SIZE) + col_qkv_w = qkv_w[:, start:end, :, :] + col_qkv_b = qkv_b[:, start:end, :] + row_linear_w = linear_w[(start * dim_head):(end * dim_head), :] + + ln_w_attr, ln_b_attr = get_param_attr(ln_w, ln_b) + qkv_w_attr, qkv_b_attr = get_param_attr(col_qkv_w, col_qkv_b) + linear_w_attr, linear_b_attr = get_param_attr(row_linear_w, linear_b) + + start = 0 if rank == 0 else (dim_ffn // MODEL_PARALLEL_SIZE) + end = start + (dim_ffn // MODEL_PARALLEL_SIZE) + col_ffn1_w = ffn1_w[:, start:end] + col_ffn1_b = ffn1_b[start:end] + row_ffn2_w = ffn2_w[start:end, :] + + ffn_ln_w_attr, ffn_ln_b_attr = get_param_attr(ffn_ln_w, ffn_ln_b) + ffn1_w_attr, ffn1_b_attr = get_param_attr(col_ffn1_w, col_ffn1_b) + ffn2_w_attr, ffn2_b_attr = get_param_attr(row_ffn2_w, ffn2_b) + + multi_transformer = FusedMultiTransformer( + hidden, + num_head, + dim_ffn, + dropout_rate=0.0, + activation="gelu", + normalize_before=True, + ln_scale_attrs=[ln_w_attr], + ln_bias_attrs=[ln_b_attr], + qkv_weight_attrs=[qkv_w_attr], + qkv_bias_attrs=[qkv_b_attr], + linear_weight_attrs=[linear_w_attr], + linear_bias_attrs=[linear_b_attr], + ffn_ln_scale_attrs=[ffn_ln_w_attr], + ffn_ln_bias_attrs=[ffn_ln_b_attr], + ffn1_weight_attrs=[ffn1_w_attr], + ffn1_bias_attrs=[ffn1_b_attr], + ffn2_weight_attrs=[ffn2_w_attr], + ffn2_bias_attrs=[ffn2_b_attr], + nranks=MODEL_PARALLEL_SIZE, + ring_id=0) + result = multi_transformer(data) + else: + ln_w_attr, ln_b_attr = get_param_attr(ln_w, ln_b) + qkv_w_attr, qkv_b_attr = get_param_attr(qkv_w, qkv_b) + linear_w_attr, linear_b_attr = get_param_attr(linear_w, linear_b) + + ffn_ln_w_attr, ffn_ln_b_attr = get_param_attr(ffn_ln_w, ffn_ln_b) + ffn1_w_attr, ffn1_b_attr = get_param_attr(ffn1_w, ffn1_b) + ffn2_w_attr, ffn2_b_attr = get_param_attr(ffn2_w, ffn2_b) + + multi_transformer = FusedMultiTransformer( + hidden, + num_head, + dim_ffn, + dropout_rate=0.0, + activation="gelu", + normalize_before=True, + ln_scale_attrs=[ln_w_attr], + ln_bias_attrs=[ln_b_attr], + qkv_weight_attrs=[qkv_w_attr], + qkv_bias_attrs=[qkv_b_attr], + linear_weight_attrs=[linear_w_attr], + linear_bias_attrs=[linear_b_attr], + ffn_ln_scale_attrs=[ffn_ln_w_attr], + ffn_ln_bias_attrs=[ffn_ln_b_attr], + ffn1_weight_attrs=[ffn1_w_attr], + ffn1_bias_attrs=[ffn1_b_attr], + ffn2_weight_attrs=[ffn2_w_attr], + ffn2_bias_attrs=[ffn2_b_attr]) + result = multi_transformer(data) + + # fused_multi_transformer have no backward + result.stop_gradient = True + predict = paddle.mean(result) + return predict + + +class TestModelParallel(TestDistRunnerBase): + def get_model(self, batch_size=2, use_dgc=False, dist_strategy=None): + # Input data + seq_len = 2 + data_in = fluid.data( + name='data_in', shape=[batch_size, seq_len, hidden], dtype=DTYPE) + + if dist_strategy: + data_loader = fluid.io.DataLoader.from_generator( + feed_list=[data_in], + capacity=64, + use_double_buffer=False, + iterable=False) + + if dist_strategy: + fleet.init(is_collective=True) + strategy = fleet.DistributedStrategy() + strategy.tensor_parallel = True + strategy.tensor_parallel_configs = {'tensor_parallel_degree': 2} + + rank = fleet.worker_index() if dist_strategy else None + avg_cost = create_model(data_in, rank) + opt = fluid.optimizer.SGD(0.1) + + if dist_strategy: + dist_opt = fleet.distributed_optimizer( + optimizer=opt, strategy=strategy) + dist_opt.minimize(avg_cost) + else: + opt.minimize(avg_cost) + + def gen_data(): + np.random.seed(2021) + while True: + data = [np.random.random([seq_len, hidden]).astype(DTYPE)] + yield data + + train_reader = paddle.batch(gen_data, batch_size=batch_size) + + if dist_strategy: + return None, avg_cost, train_reader, None, None, None, data_loader + else: + return None, avg_cost, train_reader, None, None, None + + +if __name__ == "__main__": + runtime_main(TestModelParallel) diff --git a/python/paddle/fluid/tests/unittests/test_bfgs.py b/python/paddle/fluid/tests/unittests/test_bfgs.py index c89f7205f0818..1a12913bc72e9 100644 --- a/python/paddle/fluid/tests/unittests/test_bfgs.py +++ b/python/paddle/fluid/tests/unittests/test_bfgs.py @@ -20,6 +20,10 @@ import paddle.nn.functional as F from paddle.incubate.optimizer.functional.bfgs import minimize_bfgs +from paddle.fluid.framework import _test_eager_guard + +from paddle.fluid.framework import _enable_legacy_dygraph +_enable_legacy_dygraph() np.random.seed(123) @@ -117,7 +121,7 @@ def func(x): results = test_static_graph(func, x0, dtype='float64') self.assertTrue(np.allclose(0.8, results[2])) - def test_rosenbrock(self): + def func_rosenbrock(self): # The Rosenbrock function is a standard optimization test case. a = np.random.random(size=[1]).astype('float32') minimum = [a.item(), (a**2).item()] @@ -136,6 +140,11 @@ def func(position): results = test_dynamic_graph(func, x0) self.assertTrue(np.allclose(minimum, results[2])) + def test_rosenbrock(self): + with _test_eager_guard(): + self.func_rosenbrock() + self.func_rosenbrock() + def test_exception(self): def func(x): return paddle.dot(x, x) diff --git a/python/paddle/fluid/tests/unittests/test_channel_shuffle.py b/python/paddle/fluid/tests/unittests/test_channel_shuffle.py new file mode 100644 index 0000000000000..b4a3fc387068c --- /dev/null +++ b/python/paddle/fluid/tests/unittests/test_channel_shuffle.py @@ -0,0 +1,250 @@ +# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import unittest +import numpy as np + +from op_test import OpTest +import paddle +import paddle.nn.functional as F +import paddle.fluid.core as core +import paddle.fluid as fluid + + +def channel_shuffle_np(x, groups, data_format="NCHW"): + if data_format == "NCHW": + n, c, h, w = x.shape + new_shape = (n, groups, c // groups, h, w) + npresult = np.reshape(x, new_shape) + npresult = npresult.transpose(0, 2, 1, 3, 4) + oshape = [n, c, h, w] + npresult = np.reshape(npresult, oshape) + return npresult + else: + n, h, w, c = x.shape + new_shape = (n, h, w, groups, c // groups) + npresult = np.reshape(x, new_shape) + npresult = npresult.transpose(0, 1, 2, 4, 3) + oshape = [n, h, w, c] + npresult = np.reshape(npresult, oshape) + return npresult + + +class TestChannelShuffleOp(OpTest): + def setUp(self): + self.op_type = "channel_shuffle" + self.init_data_format() + n, c, h, w = 2, 9, 4, 4 + + if self.format == "NCHW": + shape = [n, c, h, w] + if self.format == "NHWC": + shape = [n, h, w, c] + + groups = 3 + + x = np.random.random(shape).astype("float64") + npresult = channel_shuffle_np(x, groups, self.format) + + self.inputs = {'X': x} + self.outputs = {'Out': npresult} + self.attrs = {'groups': groups, "data_format": self.format} + + def init_data_format(self): + self.format = "NCHW" + + def test_check_output(self): + self.check_output() + + def test_check_grad(self): + self.check_grad(['X'], 'Out') + + +class TestChannelLast(TestChannelShuffleOp): + def init_data_format(self): + self.format = "NHWC" + + +class TestChannelShuffleAPI(unittest.TestCase): + def setUp(self): + self.x_1_np = np.random.random([2, 9, 4, 4]).astype("float64") + self.x_2_np = np.random.random([2, 4, 4, 9]).astype("float64") + self.out_1_np = channel_shuffle_np(self.x_1_np, 3) + self.out_2_np = channel_shuffle_np(self.x_2_np, 3, "NHWC") + + def test_static_graph_functional(self): + for use_cuda in ([False, True] + if core.is_compiled_with_cuda() else [False]): + place = paddle.CUDAPlace(0) if use_cuda else paddle.CPUPlace() + + paddle.enable_static() + x_1 = paddle.fluid.data( + name="x", shape=[2, 9, 4, 4], dtype="float64") + x_2 = paddle.fluid.data( + name="x2", shape=[2, 4, 4, 9], dtype="float64") + out_1 = F.channel_shuffle(x_1, 3) + out_2 = F.channel_shuffle(x_2, 3, "NHWC") + + exe = paddle.static.Executor(place=place) + res_1 = exe.run(fluid.default_main_program(), + feed={"x": self.x_1_np}, + fetch_list=out_1, + use_prune=True) + + res_2 = exe.run(fluid.default_main_program(), + feed={"x2": self.x_2_np}, + fetch_list=out_2, + use_prune=True) + + assert np.allclose(res_1, self.out_1_np) + assert np.allclose(res_2, self.out_2_np) + + # same test between layer and functional in this op. + def test_static_graph_layer(self): + for use_cuda in ([False, True] + if core.is_compiled_with_cuda() else [False]): + place = paddle.CUDAPlace(0) if use_cuda else paddle.CPUPlace() + + paddle.enable_static() + x_1 = paddle.fluid.data( + name="x", shape=[2, 9, 4, 4], dtype="float64") + x_2 = paddle.fluid.data( + name="x2", shape=[2, 4, 4, 9], dtype="float64") + # init instance + ps_1 = paddle.nn.ChannelShuffle(3) + ps_2 = paddle.nn.ChannelShuffle(3, "NHWC") + out_1 = ps_1(x_1) + out_2 = ps_2(x_2) + out_1_np = channel_shuffle_np(self.x_1_np, 3) + out_2_np = channel_shuffle_np(self.x_2_np, 3, "NHWC") + + exe = paddle.static.Executor(place=place) + res_1 = exe.run(fluid.default_main_program(), + feed={"x": self.x_1_np}, + fetch_list=out_1, + use_prune=True) + + res_2 = exe.run(fluid.default_main_program(), + feed={"x2": self.x_2_np}, + fetch_list=out_2, + use_prune=True) + + assert np.allclose(res_1, out_1_np) + assert np.allclose(res_2, out_2_np) + + def run_dygraph(self, groups, data_format): + + n, c, h, w = 2, 9, 4, 4 + + if data_format == "NCHW": + shape = [n, c, h, w] + if data_format == "NHWC": + shape = [n, h, w, c] + + x = np.random.random(shape).astype("float64") + + npresult = channel_shuffle_np(x, groups, data_format) + + for use_cuda in ([False, True] + if core.is_compiled_with_cuda() else [False]): + place = paddle.CUDAPlace(0) if use_cuda else paddle.CPUPlace() + + paddle.disable_static(place=place) + + channel_shuffle = paddle.nn.ChannelShuffle( + groups, data_format=data_format) + result = channel_shuffle(paddle.to_tensor(x)) + + self.assertTrue(np.allclose(result.numpy(), npresult)) + + result_functional = F.channel_shuffle( + paddle.to_tensor(x), 3, data_format) + self.assertTrue(np.allclose(result_functional.numpy(), npresult)) + + channel_shuffle_str = 'groups={}'.format(groups) + if data_format != 'NCHW': + channel_shuffle_str += ', data_format={}'.format(data_format) + self.assertEqual(channel_shuffle.extra_repr(), channel_shuffle_str) + + def test_dygraph1(self): + self.run_dygraph(3, "NCHW") + + def test_dygraph2(self): + self.run_dygraph(3, "NHWC") + + +class TestChannelShuffleError(unittest.TestCase): + def test_error_functional(self): + def error_input(): + with paddle.fluid.dygraph.guard(): + x = np.random.random([9, 4, 4]).astype("float64") + channel_shuffle = F.channel_shuffle(paddle.to_tensor(x), 3) + + self.assertRaises(ValueError, error_input) + + def error_groups_1(): + with paddle.fluid.dygraph.guard(): + x = np.random.random([2, 9, 4, 4]).astype("float64") + channel_shuffle = F.channel_shuffle(paddle.to_tensor(x), 3.33) + + self.assertRaises(TypeError, error_groups_1) + + def error_groups_2(): + with paddle.fluid.dygraph.guard(): + x = np.random.random([2, 9, 4, 4]).astype("float64") + channel_shuffle = F.channel_shuffle(paddle.to_tensor(x), -1) + + self.assertRaises(ValueError, error_groups_2) + + def error_data_format(): + with paddle.fluid.dygraph.guard(): + x = np.random.random([2, 9, 4, 4]).astype("float64") + channel_shuffle = F.channel_shuffle( + paddle.to_tensor(x), 3, "WOW") + + self.assertRaises(ValueError, error_data_format) + + def test_error_layer(self): + def error_input_layer(): + with paddle.fluid.dygraph.guard(): + x = np.random.random([9, 4, 4]).astype("float64") + cs = paddle.nn.ChannelShuffle(3) + cs(paddle.to_tensor(x)) + + self.assertRaises(ValueError, error_input_layer) + + def error_groups_layer_1(): + with paddle.fluid.dygraph.guard(): + x = np.random.random([2, 9, 4, 4]).astype("float64") + cs = paddle.nn.ChannelShuffle(3.33) + + self.assertRaises(TypeError, error_groups_layer_1) + + def error_groups_layer_2(): + with paddle.fluid.dygraph.guard(): + x = np.random.random([2, 9, 4, 4]).astype("float64") + cs = paddle.nn.ChannelShuffle(-1) + + self.assertRaises(ValueError, error_groups_layer_2) + + def error_data_format_layer(): + with paddle.fluid.dygraph.guard(): + x = np.random.random([2, 9, 4, 4]).astype("float64") + cs = paddle.nn.ChannelShuffle(3, "MEOW") + + self.assertRaises(ValueError, error_data_format_layer) + + +if __name__ == '__main__': + unittest.main() diff --git a/python/paddle/fluid/tests/unittests/test_clip_op.py b/python/paddle/fluid/tests/unittests/test_clip_op.py index 37b1cfd02faf7..121b91d741546 100644 --- a/python/paddle/fluid/tests/unittests/test_clip_op.py +++ b/python/paddle/fluid/tests/unittests/test_clip_op.py @@ -200,7 +200,7 @@ def test_clip(self): np.allclose(res11, (data * 10).astype(np.int64).clip(2, 8))) paddle.disable_static() - def test_clip_dygraph(self): + def func_clip_dygraph(self): paddle.disable_static() place = fluid.CUDAPlace(0) if fluid.core.is_compiled_with_cuda( ) else fluid.CPUPlace() @@ -233,9 +233,29 @@ def test_clip_dygraph(self): np.allclose(out_5.numpy(), (data * 10).astype(np.int64).clip(2, 8))) self.assertTrue(np.allclose(out_6.numpy(), data.clip(0.2, 0.8))) - def test_eager(self): + def test_clip_dygraph(self): + with _test_eager_guard(): + self.func_clip_dygraph() + self.func_clip_dygraph() + + def test_clip_dygraph_default_max(self): + paddle.disable_static() with _test_eager_guard(): - self.test_clip_dygraph() + x_int32 = paddle.to_tensor([1, 2, 3], dtype="int32") + x_int64 = paddle.to_tensor([1, 2, 3], dtype="int64") + x_f32 = paddle.to_tensor([1, 2, 3], dtype="float32") + egr_out1 = paddle.clip(x_int32, min=1) + egr_out2 = paddle.clip(x_int64, min=1) + egr_out3 = paddle.clip(x_f32, min=1) + x_int32 = paddle.to_tensor([1, 2, 3], dtype="int32") + x_int64 = paddle.to_tensor([1, 2, 3], dtype="int64") + x_f32 = paddle.to_tensor([1, 2, 3], dtype="float32") + out1 = paddle.clip(x_int32, min=1) + out2 = paddle.clip(x_int64, min=1) + out3 = paddle.clip(x_f32, min=1) + self.assertTrue(np.allclose(out1.numpy(), egr_out1.numpy())) + self.assertTrue(np.allclose(out2.numpy(), egr_out2.numpy())) + self.assertTrue(np.allclose(out3.numpy(), egr_out3.numpy())) def test_errors(self): paddle.enable_static() diff --git a/python/paddle/fluid/tests/unittests/test_dataloader_autotune.py b/python/paddle/fluid/tests/unittests/test_dataloader_autotune.py new file mode 100755 index 0000000000000..7348783bd6748 --- /dev/null +++ b/python/paddle/fluid/tests/unittests/test_dataloader_autotune.py @@ -0,0 +1,103 @@ +# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import print_function +import unittest +import numpy as np +import tempfile +import warnings +import json +import paddle +import paddle.nn as nn +from paddle.io import Dataset, DataLoader, BatchSampler, SequenceSampler +import sys +import os + + +class RandomDataset(Dataset): + def __init__(self, num_samples): + self.num_samples = num_samples + + def __getitem__(self, idx): + image = np.random.random([10]).astype('float32') + label = np.random.randint(0, 10 - 1, (1, )).astype('int64') + return image, label + + def __len__(self): + return self.num_samples + + +class SimpleNet(nn.Layer): + def __init__(self): + super(SimpleNet, self).__init__() + self.fc = nn.Linear(10, 10) + + def forward(self, image): + return self.fc(image) + + +class TestAutoTune(unittest.TestCase): + def setUp(self): + self.batch_size = 1 + self.dataset = RandomDataset(10) + + def test_dataloader_use_autotune(self): + paddle.incubate.autotune.set_config( + config={"dataloader": { + "enable": True, + "tuning_steps": 1, + }}) + loader = DataLoader( + self.dataset, batch_size=self.batch_size, num_workers=0) + + def test_dataloader_disable_autotune(self): + config = {"dataloader": {"enable": False, "tuning_steps": 1}} + tfile = tempfile.NamedTemporaryFile(mode="w+", delete=False) + json.dump(config, tfile) + tfile.close() + paddle.incubate.autotune.set_config(tfile.name) + os.remove(tfile.name) + loader = DataLoader( + self.dataset, batch_size=self.batch_size, num_workers=2) + if (sys.platform == 'darwin' or sys.platform == 'win32'): + self.assertEqual(loader.num_workers, 0) + else: + self.assertEqual(loader.num_workers, 2) + + def test_distributer_batch_sampler_autotune(self): + paddle.incubate.autotune.set_config( + config={"dataloader": { + "enable": True, + "tuning_steps": 1, + }}) + batch_sampler = paddle.io.DistributedBatchSampler( + self.dataset, batch_size=self.batch_size) + loader = DataLoader( + self.dataset, batch_sampler=batch_sampler, num_workers=2) + + +class TestAutoTuneAPI(unittest.TestCase): + def test_set_config_warnings(self): + with warnings.catch_warnings(record=True) as w: + config = {"kernel": {"enable": 1, "tuning_range": True}} + tfile = tempfile.NamedTemporaryFile(mode="w+", delete=False) + json.dump(config, tfile) + tfile.close() + paddle.incubate.autotune.set_config(tfile.name) + os.remove(tfile.name) + self.assertTrue(len(w) == 2) + + +if __name__ == '__main__': + unittest.main() diff --git a/python/paddle/fluid/tests/unittests/test_diff_op.py b/python/paddle/fluid/tests/unittests/test_diff_op.py index 4a96827bd7c3c..99a46bfd9584d 100644 --- a/python/paddle/fluid/tests/unittests/test_diff_op.py +++ b/python/paddle/fluid/tests/unittests/test_diff_op.py @@ -19,6 +19,8 @@ import paddle.fluid as fluid import paddle.fluid.layers as layers import paddle.fluid.core as core +from paddle.fluid.framework import _enable_legacy_dygraph +_enable_legacy_dygraph() class TestDiffOp(unittest.TestCase): diff --git a/python/paddle/fluid/tests/unittests/test_dist_fleet_ctr.py b/python/paddle/fluid/tests/unittests/test_dist_fleet_ctr.py index 59d196fdf55e5..09d64a318d6d8 100644 --- a/python/paddle/fluid/tests/unittests/test_dist_fleet_ctr.py +++ b/python/paddle/fluid/tests/unittests/test_dist_fleet_ctr.py @@ -39,6 +39,8 @@ def check_with_place(self, "http_proxy": "", "CPU_NUM": "2", "LOG_DIRNAME": "/tmp", + "SAVE_CACHE_DIRNAME": + "/tmp/TestDistMnistAsyncInMemoryDataset2x2/cache_model", "LOG_PREFIX": self.__class__.__name__, } diff --git a/python/paddle/fluid/tests/unittests/test_distributed_fused_lamb_op_with_clip.py b/python/paddle/fluid/tests/unittests/test_distributed_fused_lamb_op_with_clip.py index af99529adfa74..315580dd31ad7 100644 --- a/python/paddle/fluid/tests/unittests/test_distributed_fused_lamb_op_with_clip.py +++ b/python/paddle/fluid/tests/unittests/test_distributed_fused_lamb_op_with_clip.py @@ -34,7 +34,9 @@ def remove_file_if_exists(file_name): shutil.rmtree(file_name) -def run_test(clip_after_allreduce=True, max_global_norm=-1.0): +def run_test(clip_after_allreduce=True, + max_global_norm=-1.0, + gradient_merge_steps=1): if not paddle.is_compiled_with_cuda(): return if os.name == 'nt': @@ -55,6 +57,7 @@ def run_test(clip_after_allreduce=True, max_global_norm=-1.0): os.environ['CLIP_AFTER_ALLREDUCE'] = str(clip_after_allreduce) os.environ['MAX_GLOBAL_NORM'] = str(max_global_norm) + os.environ['GRADIENT_MERGE_STEPS'] = str(gradient_merge_steps) touch_file_env = 'SUCCESS_TOUCH_FILE' touch_file_name = 'distributed_fused_lamb_touch_file_{}'.format(os.getpid()) diff --git a/python/paddle/fluid/tests/unittests/test_distributed_fused_lamb_op_with_gradient_merge.py b/python/paddle/fluid/tests/unittests/test_distributed_fused_lamb_op_with_gradient_merge.py new file mode 100644 index 0000000000000..1822b77d0d0e5 --- /dev/null +++ b/python/paddle/fluid/tests/unittests/test_distributed_fused_lamb_op_with_gradient_merge.py @@ -0,0 +1,28 @@ +# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from test_distributed_fused_lamb_op_with_clip import run_test +import unittest + + +class TestDistributedFusedLambGradientMerge(unittest.TestCase): + def test_gm(self): + run_test( + clip_after_allreduce=True, + max_global_norm=-1.0, + gradient_merge_steps=2) + + +if __name__ == "__main__": + unittest.main() diff --git a/python/paddle/fluid/tests/unittests/test_dropout_op.py b/python/paddle/fluid/tests/unittests/test_dropout_op.py index 3aca428ac77af..20abeaec7268c 100644 --- a/python/paddle/fluid/tests/unittests/test_dropout_op.py +++ b/python/paddle/fluid/tests/unittests/test_dropout_op.py @@ -22,7 +22,8 @@ import paddle.static as static import paddle.fluid as fluid from paddle.fluid import Program, program_guard -from paddle.fluid.framework import _test_eager_guard +from paddle.fluid.framework import _test_eager_guard, _enable_legacy_dygraph +_enable_legacy_dygraph() import os from paddle import _C_ops @@ -951,6 +952,7 @@ def cal_grad_downscale_in_infer(self, mask): return mask.astype("float32") def test_backward_downscale_in_infer(self): + _enable_legacy_dygraph() for place in self.places: with fluid.dygraph.guard(place): diff --git a/python/paddle/fluid/tests/unittests/test_dygraph_mnist_fp16.py b/python/paddle/fluid/tests/unittests/test_dygraph_mnist_fp16.py index 7503a9172fc21..6c2516d6c11ef 100644 --- a/python/paddle/fluid/tests/unittests/test_dygraph_mnist_fp16.py +++ b/python/paddle/fluid/tests/unittests/test_dygraph_mnist_fp16.py @@ -19,6 +19,7 @@ import paddle.fluid as fluid from paddle.fluid.dygraph.nn import Conv2D, Pool2D, Linear +from paddle.fluid.framework import _test_eager_guard class SimpleImgConvPool(fluid.dygraph.Layer): @@ -117,7 +118,7 @@ def forward(self, inputs, label): class TestMnist(unittest.TestCase): - def test_mnist_fp16(self): + def func_mnist_fp16(self): if not fluid.is_compiled_with_cuda(): return x = np.random.randn(1, 3, 224, 224).astype("float16") @@ -129,6 +130,11 @@ def test_mnist_fp16(self): loss = model(x, y) print(loss.numpy()) + def test_mnist_fp16(self): + with _test_eager_guard(): + self.func_mnist_fp16() + self.func_mnist_fp16() + if __name__ == "__main__": unittest.main() diff --git a/python/paddle/fluid/tests/unittests/test_eigh_op.py b/python/paddle/fluid/tests/unittests/test_eigh_op.py index 2abbcc98a6b7e..9c9cd883313a2 100644 --- a/python/paddle/fluid/tests/unittests/test_eigh_op.py +++ b/python/paddle/fluid/tests/unittests/test_eigh_op.py @@ -19,6 +19,8 @@ import paddle from op_test import OpTest from gradient_checker import grad_check +from paddle.fluid.framework import _enable_legacy_dygraph +_enable_legacy_dygraph() def valid_eigh_result(A, eigh_value, eigh_vector, uplo): diff --git a/python/paddle/fluid/tests/unittests/test_einsum_op.py b/python/paddle/fluid/tests/unittests/test_einsum_op.py new file mode 100644 index 0000000000000..565e43214ea32 --- /dev/null +++ b/python/paddle/fluid/tests/unittests/test_einsum_op.py @@ -0,0 +1,151 @@ +# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import print_function + +import unittest +import numpy as np +import paddle +from op_test import OpTest + + +class TestEinsumBinary(OpTest): + def setUp(self): + paddle.enable_static() + self.op_type = "einsum" + self.disable = False + self.set_mandatory() + self.init_input() + np.random.seed(123) + out = np.einsum(self.equation, *self.inputs) + self.operands = [] + for idx, inp in enumerate(self.inputs): + self.operands.append(("x" + str(idx), inp)) + self.inputs = {"Operands": self.operands} + self.attrs = {"equation": self.equation} + self.outputs = {'Out': out} + + def init_input(self): + self.inputs = [] + for t, s in zip(self.types, self.shapes): + self.inputs.append(np.random.random(s).astype(t)) + + def set_mandatory(self): + self.disable = False + self.shapes = [(10, 10, 20), (20, 6)] + self.types = [np.float64, np.float64] + self.equation = "mij,jk->ki" + + def test_check_output(self): + if not self.disable: + self.check_output() + + def test_grad(self): + if not self.disable: + self.check_grad([op[0] for op in self.operands], ["Out"]) + + +class TestEinsum1(TestEinsumBinary): + def set_mandatory(self): + self.shapes = [(20, 3, 3), (20, 3, 3)] + self.types = [np.float64, np.float64] + self.equation = "mij,mjk->mik" + + +class TestEinsum2(TestEinsumBinary): + def set_mandatory(self): + self.shapes = [(20, 3, 3), (20, 3, 3)] + self.types = [np.float64, np.float64] + self.equation = "mij,mjk->ikm" + + +class TestEinsum3(TestEinsumBinary): + def set_mandatory(self): + self.shapes = [(10, 10), (10, 10)] + self.types = [np.float64, np.float64] + self.equation = "ij,jk->ik" # }}} + + +class TestEinsumWithReduction(TestEinsumBinary): + def set_mandatory(self): + self.shapes = [(10, 3, 5), (5, 30)] + self.types = [np.float64, np.float64] + self.equation = "ijk,kl->jl" + + +class TestEinsumWithReduction1(TestEinsumBinary): + def set_mandatory(self): + self.shapes = [(10, 3, 3, 5), (10, 5, 10, 10)] + self.types = [np.float64, np.float64] + self.equation = "mijk,mklh->ljm" + + +class TestEinsumWithUnary(TestEinsumBinary): + def set_mandatory(self): + self.shapes = [(10, 10, 3, 5)] + self.types = [np.float64] + self.equation = "mijk->mi" + + +class TestEinsumWithUnary1(TestEinsumBinary): + def set_mandatory(self): + self.shapes = [(5, 10, 3, 3), (3, 6, 3, 10)] + self.types = [np.float64, np.float64] + self.equation = "imjl,jklm->imk" + + +class TestEinsumWithBroadcast1(TestEinsumBinary): + def set_mandatory(self): + self.shapes = [(5, 10, 3, 3)] + self.types = [np.float64] + self.equation = "i...->..." + + +class TestEinsumWithBroadcast2(TestEinsumBinary): + def set_mandatory(self): + self.shapes = [(10, 11), (3, 4, 5, 10)] + self.types = [np.float64, np.float64] + self.equation = "...ij,...i->j..." + + +class TestEinsumWithBroadcast3(TestEinsumBinary): + def set_mandatory(self): + self.shapes = [(10, 3, 2, 3, 4), (12, 10)] + self.types = [np.float64, np.float64] + self.equation = "k...,...jk->...k" + + +class TestEinsumWithBroadcast4(TestEinsumBinary): + def set_mandatory(self): + self.shapes = [(10, 3, 2, 3, 4), (12, 10)] + self.types = [np.float64, np.float64] + self.equation = "a...d,...cb->...abcd" + + +class TestEinsumWithBroadcast5(TestEinsumBinary): + def set_mandatory(self): + self.shapes = [(3, 2, 2, 10), (10, 3, 2, 2)] + self.types = [np.float64, np.float64] + self.equation = "...a,a...->..." + + +class TestEinsumWithBroadcast6(TestEinsumBinary): + def set_mandatory(self): + self.shapes = [(100), (100)] + self.types = [np.float64, np.float64] + self.equation = "i,i->" + + +if __name__ == "__main__": + unittest.main() diff --git a/python/paddle/fluid/tests/unittests/test_elementwise_nn_grad.py b/python/paddle/fluid/tests/unittests/test_elementwise_nn_grad.py index c51c8098706a6..8f6f9851c7006 100644 --- a/python/paddle/fluid/tests/unittests/test_elementwise_nn_grad.py +++ b/python/paddle/fluid/tests/unittests/test_elementwise_nn_grad.py @@ -17,6 +17,7 @@ import unittest import numpy as np +import paddle import paddle.fluid as fluid import paddle.fluid.layers as layers import paddle.fluid.core as core @@ -45,6 +46,7 @@ def func(self, place): [x, y], out, x_init=[x_arr, y_arr], place=place, eps=eps) def test_grad(self): + paddle.enable_static() places = [fluid.CPUPlace()] if core.is_compiled_with_cuda(): places.append(fluid.CUDAPlace(0)) @@ -72,6 +74,7 @@ def func(self, place): [x, y], out, x_init=[x_arr, y_arr], place=place, eps=eps) def test_grad(self): + paddle.enable_static() places = [fluid.CPUPlace()] if core.is_compiled_with_cuda(): places.append(fluid.CUDAPlace(0)) @@ -99,6 +102,7 @@ def func(self, place): [x, y], out, x_init=[x_arr, y_arr], place=place, eps=eps) def test_grad(self): + paddle.enable_static() places = [fluid.CPUPlace()] if core.is_compiled_with_cuda(): places.append(fluid.CUDAPlace(0)) @@ -126,6 +130,7 @@ def func(self, place): [x, y], out, x_init=[x_arr, y_arr], place=place, eps=eps) def test_grad(self): + paddle.enable_static() places = [fluid.CPUPlace()] if core.is_compiled_with_cuda(): places.append(fluid.CUDAPlace(0)) @@ -153,6 +158,7 @@ def func(self, place): [x, y], out, x_init=[x_arr, y_arr], place=place, eps=eps) def test_grad(self): + paddle.enable_static() places = [fluid.CPUPlace()] if core.is_compiled_with_cuda(): places.append(fluid.CUDAPlace(0)) @@ -180,6 +186,7 @@ def func(self, place): [x, y], out, x_init=[x_arr, y_arr], place=place, eps=eps) def test_grad(self): + paddle.enable_static() places = [fluid.CPUPlace()] if core.is_compiled_with_cuda(): places.append(fluid.CUDAPlace(0)) @@ -208,6 +215,7 @@ def func(self, place): [x, y], out, x_init=[x_arr, y_arr], place=place, eps=eps, atol=1e-3) def test_grad(self): + paddle.enable_static() places = [fluid.CPUPlace()] if core.is_compiled_with_cuda(): places.append(fluid.CUDAPlace(0)) @@ -236,6 +244,7 @@ def func(self, place): [x, y], out, x_init=[x_arr, y_arr], place=place, eps=eps, atol=1e-3) def test_grad(self): + paddle.enable_static() places = [fluid.CPUPlace()] if core.is_compiled_with_cuda(): places.append(fluid.CUDAPlace(0)) @@ -263,6 +272,7 @@ def func(self, place): [x, y], out, x_init=[x_arr, y_arr], place=place, eps=eps) def test_grad(self): + paddle.enable_static() places = [fluid.CPUPlace()] if core.is_compiled_with_cuda(): places.append(fluid.CUDAPlace(0)) @@ -290,6 +300,7 @@ def func(self, place): [x, y], out, x_init=[x_arr, y_arr], place=place, eps=eps) def test_grad(self): + paddle.enable_static() places = [fluid.CPUPlace()] if core.is_compiled_with_cuda(): places.append(fluid.CUDAPlace(0)) @@ -298,6 +309,9 @@ def test_grad(self): class TestElementwiseMulTripleGradCheck(unittest.TestCase): + def multiply_wrapper(self, x): + return paddle.multiply(x[0], x[1]) + @prog_scope() def func(self, place): # the shape of input variable should be clearly specified, not inlcude -1. @@ -315,8 +329,14 @@ def func(self, place): gradient_checker.triple_grad_check( [x, y], out, x_init=[x_arr, y_arr], place=place, eps=eps) + gradient_checker.triple_grad_check_for_dygraph( + self.multiply_wrapper, [x, y], + out, + x_init=[x_arr, y_arr], + place=place) def test_grad(self): + paddle.enable_static() places = [fluid.CPUPlace()] if core.is_compiled_with_cuda(): places.append(fluid.CUDAPlace(0)) @@ -344,6 +364,7 @@ def func(self, place): [x, y], out, x_init=[x_arr, y_arr], place=place, eps=eps) def test_grad(self): + paddle.enable_static() places = [fluid.CPUPlace()] if core.is_compiled_with_cuda(): places.append(fluid.CUDAPlace(0)) diff --git a/python/paddle/fluid/tests/unittests/test_faster_tokenizer_op.py b/python/paddle/fluid/tests/unittests/test_faster_tokenizer_op.py index 190345958e0e5..87c4656cfa809 100755 --- a/python/paddle/fluid/tests/unittests/test_faster_tokenizer_op.py +++ b/python/paddle/fluid/tests/unittests/test_faster_tokenizer_op.py @@ -22,7 +22,8 @@ import paddle import paddle.nn as nn from paddle.dataset.common import DATA_HOME -from paddle.fluid.framework import core, _non_static_mode +from paddle.fluid.framework import core, _non_static_mode, _enable_legacy_dygraph +_enable_legacy_dygraph() from paddle.fluid.layer_helper import LayerHelper from paddle import _C_ops diff --git a/python/paddle/fluid/tests/unittests/test_fuse_gemm_epilogue_pass.py b/python/paddle/fluid/tests/unittests/test_fuse_gemm_epilogue_pass.py index 7f3180e21d8c6..00d91b1fab0f1 100644 --- a/python/paddle/fluid/tests/unittests/test_fuse_gemm_epilogue_pass.py +++ b/python/paddle/fluid/tests/unittests/test_fuse_gemm_epilogue_pass.py @@ -49,8 +49,8 @@ def verify_node_count(graph, node_name, target_count): class MultiFCLayer(paddle.nn.Layer): def __init__(self, hidden, Activation): super(MultiFCLayer, self).__init__() - self.linear1 = paddle.nn.Linear(hidden, hidden) - self.linear2 = paddle.nn.Linear(hidden, hidden) + self.linear1 = paddle.nn.Linear(hidden, 4 * hidden) + self.linear2 = paddle.nn.Linear(4 * hidden, hidden) self.linear3 = paddle.nn.Linear(hidden, hidden) self.relu1 = Activation() diff --git a/python/paddle/fluid/tests/unittests/test_fused_multi_transformer_op.py b/python/paddle/fluid/tests/unittests/test_fused_multi_transformer_op.py new file mode 100644 index 0000000000000..8f77972de8656 --- /dev/null +++ b/python/paddle/fluid/tests/unittests/test_fused_multi_transformer_op.py @@ -0,0 +1,542 @@ +# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import numpy as np + +import paddle +import paddle.nn as nn +import paddle.fluid.core as core +import paddle.nn.functional as F +import paddle.incubate.nn.functional as incubate_f +from paddle.nn.layer.norm import LayerNorm +from paddle.nn.layer.common import Linear, Dropout +from paddle.nn.layer.transformer import _convert_attention_mask +from paddle import tensor +from paddle.fluid import layers +import unittest +from op_test import OpTest +from paddle.fluid.framework import default_main_program +from paddle.fluid.dygraph.layers import Layer +from paddle.fluid.layer_helper import LayerHelper +from paddle.nn.initializer import Constant +from paddle.fluid.data_feeder import check_variable_and_dtype, check_dtype +from paddle.fluid.framework import _non_static_mode, default_main_program +from paddle import _C_ops +from paddle.incubate.nn.functional import fused_multi_transformer + +default_main_program().random_seed = 42 + + +class TestFusedMultiTransformerOp(OpTest): + def setUp(self): + self.config() + self.generate_input_data() + + self.rtol = 1e-5 + # FIXME(wangxi): Because there is a problem with the test precision + # on A100, atol is temporarily set to 1e-2, and it will be + # changed back after the precision problem is solved. + self.atol = 1e-2 + # make sure local development precision + if "V100" in paddle.device.cuda.get_device_name(): + self.atol = 1e-4 + if self.x_type is np.float16: + self.atol = 1e-1 + + paddle.set_default_dtype(self.x_type) + self.__class__.op_type = "fused_multi_transformer" + # use autograd to check grad in this unittest. + self.__class__.no_need_check_grad = False + + bias_attr = paddle.fluid.ParamAttr( + initializer=paddle.fluid.initializer.Constant(value=0.0005)) + self.q_proj = Linear( + self.embed_dim, + self.embed_dim, + self.weight_attr, + bias_attr=bias_attr) + #bias_attr=self.bias_attr) + + self.k_proj = Linear( + self.kdim, + self.embed_dim, + self.weight_attr, + bias_attr=self.bias_attr) + self.v_proj = Linear( + self.vdim, + self.embed_dim, + self.weight_attr, + bias_attr=self.bias_attr) + self.out_proj = Linear( + self.embed_dim, + self.embed_dim, + self.weight_attr, + bias_attr=self.bias_attr) + + self.ffn1_proj = Linear( + self.embed_dim, + 4 * self.embed_dim, + self.weight_attr, + bias_attr=self.bias_attr) + self.ffn2_proj = Linear( + 4 * self.embed_dim, + self.embed_dim, + self.weight_attr, + bias_attr=self.bias_attr) + + paddle.set_default_dtype(np.float32) + self.norm = LayerNorm(self.embed_dim) + self.ffn_norm = LayerNorm(self.embed_dim) + + paddle.set_default_dtype(self.x_type) + self.dropout = Dropout(self.dropout_prob, mode="upscale_in_train") + self.activation = getattr(F, self.act_method) + + def config(self): + # for debug + self.debug = False + + self.x_type = np.float32 + self.attn_mask_type = np.float64 + self.pre_layer_norm = True + self.has_attn_mask = True + + # has_cache_kv, gen_cache_kv, stage + # False, False, not generation + # True, True, generation context stage + # True, False, generation decoder stage + self.has_cache_kv = False + self.gen_cache_kv = False + + self.training = False + + self.layers = 4 + self.batch_size = 8 + self.query_length = 128 + self.cache_length = 128 + self.head_dim = 64 + self.num_heads = 16 + self.embed_dim = self.head_dim * self.num_heads + + self.dropout_prob = 0.0 + self.attn_dropout_prob = 0.0 + self.act_method = 'gelu' + self.weight_attr = None + self.bias_attr = None + self.kdim, self.vdim = self.embed_dim, self.embed_dim + self.key_length, self.value_length = self.query_length, self.query_length + + def generate_input_data(self): + self.query = np.random.rand(self.batch_size, self.query_length, + self.embed_dim).astype(self.x_type) + out_seq_len = self.key_length + if self.has_cache_kv: + assert self.training is False, ValueError( + 'cache_kv can only used in inference') + self.cache_kv = np.random.rand(2, self.batch_size, self.num_heads, + self.cache_length, + self.head_dim).astype(self.x_type) + if self.gen_cache_kv: + self.cache_kv[:] = 0 + else: + out_seq_len += self.cache_length + else: + self.cache_kv = None + + if self.has_attn_mask: + # [B, n_head, seq_len, out_seq_len] + self.attn_mask = np.ones( + (self.batch_size, 1, self.query_length, out_seq_len), + dtype=self.attn_mask_type) + if self.attn_mask_type == np.int64: + self.attn_mask = np.tril(self.attn_mask) + elif self.attn_mask_type == np.float64: + if self.has_cache_kv and not self.gen_cache_kv: + # NOTE: decoder stage, -1(out_seq_len) should no mask + self.attn_mask[:, :, :, -2] = 0.0 + self.attn_mask = (self.attn_mask - 1.0) * 1e4 + else: + self.attn_mask = (np.tril(self.attn_mask) - 1.0) * 1e4 + else: + raise ValueError( + "'attn_mask_type' should be 'int64' or 'float64'.") + else: + self.attn_mask = None + self.key, self.value = self.query, self.query + + self.dout = np.random.random((self.batch_size, self.query_length, + self.embed_dim)).astype(self.x_type) + + def GetBaselineOut(self): + paddle.disable_static(place=paddle.CUDAPlace(0)) + tensor_query = paddle.to_tensor(self.query, stop_gradient=False) + + cache_kvs = [] + cache_kv = None + if self.has_cache_kv: + cache_kv = paddle.to_tensor(self.cache_kv, stop_gradient=False) + + if self.has_attn_mask: + attn_mask = paddle.to_tensor(self.attn_mask, stop_gradient=False) + else: + attn_mask = None + + for i in range(self.layers): + residual = tensor_query + ln1_out = tensor_query + if self.pre_layer_norm: + ln1_out = self.norm(tensor_query) + + q = self.q_proj(ln1_out) + q = tensor.reshape(x=q, shape=[0, 0, self.num_heads, self.head_dim]) + q_out = tensor.transpose(x=q, perm=[0, 2, 1, 3]) + k = self.k_proj(ln1_out) + v = self.v_proj(ln1_out) + k = tensor.reshape(x=k, shape=[0, 0, self.num_heads, self.head_dim]) + k_out = tensor.transpose(x=k, perm=[0, 2, 1, 3]) + v = tensor.reshape(x=v, shape=[0, 0, self.num_heads, self.head_dim]) + v_out = tensor.transpose(x=v, perm=[0, 2, 1, 3]) + + if self.has_cache_kv: + # [1, B, n_head, cache_seq_len, head_dim] + cache_k, cache_v = paddle.split(cache_kv, 2) + cache_k = paddle.squeeze(cache_k, axis=0) + cache_v = paddle.squeeze(cache_v, axis=0) + # [B, n_head, cache_seq_len + seq_len, head_dim] + # out_seq_len = cache_seq_len + seq_len + if self.debug: + print('q out is') + print(q_out[0, 0, :, :]) + print('cache k out seq=128') + print(k_out[0, 0, :, :]) + if self.gen_cache_kv: + cache_kvs.append((k_out, v_out)) + else: + k_out = paddle.concat([cache_k, k_out], axis=-2) + v_out = paddle.concat([cache_v, v_out], axis=-2) + + # [B, n_head, seq_len, head_dim] * [B, n_head, out_seq_len, head_dim] + # --> [B, n_head, seq_len, out_seq_len] + qk_out = layers.matmul( + x=q_out, y=k_out, transpose_y=True, alpha=self.head_dim**-0.5) + + if self.debug: + print('qk out is') + print(qk_out[0][0][0]) + + if attn_mask is not None: + attn_mask = _convert_attention_mask(attn_mask, qk_out.dtype) + attn_mask_out = qk_out + attn_mask + if self.debug: + print('attn mask out is') + print(attn_mask_out[0][0][0]) + softmax_out = F.softmax(attn_mask_out) + else: + softmax_out = F.softmax(qk_out) + + if self.debug: + print('softmax out is') + print(softmax_out[0][0][0]) + if self.dropout_prob: + dropout_out = F.dropout( + softmax_out, + self.dropout_prob, + training=self.training, + mode="upscale_in_train") + # [B, n_head, seq_len, out_seq_len] * [B, n_head, out_seq_len, head_dim] + # --> [B, n_head, seq_len, head_dim] + qktv_out = tensor.matmul(dropout_out, v_out) + else: + qktv_out = tensor.matmul(softmax_out, v_out) + + fmha_out = tensor.transpose(qktv_out, perm=[0, 2, 1, 3]) + if self.debug: + print('fmha out is') + print(fmha_out[0][0][0]) + out_linear_in = tensor.reshape( + x=fmha_out, + shape=[0, 0, fmha_out.shape[2] * fmha_out.shape[3]]) + out = self.out_proj(out_linear_in) + + residual_out = residual + self.dropout(out) + if not self.pre_layer_norm: + attn_out = self.norm(residual_out) + else: + attn_out = residual_out + + ffn_ln_out = attn_out + if self.pre_layer_norm: + ffn_ln_out = self.ffn_norm(attn_out) + + ffn1_out = self.ffn1_proj(ffn_ln_out) + ffn1_out = self.dropout(self.activation(ffn1_out)) + ffn2_out = self.ffn2_proj(ffn1_out) + + residual_out = attn_out + self.dropout(ffn2_out) + final_out = residual_out + if not self.pre_layer_norm: + final_out = self.ffn_norm(residual_out) + + tensor_query = final_out + + if self.has_cache_kv and self.gen_cache_kv: + return final_out, cache_kvs + return final_out + + def GetFusedMultiTransformerOut(self): + paddle.disable_static(place=paddle.CUDAPlace(0)) + q_proj_weight = paddle.to_tensor( + self.q_proj.weight, stop_gradient=False) + k_proj_weight = paddle.to_tensor( + self.k_proj.weight, stop_gradient=False) + v_proj_weight = paddle.to_tensor( + self.v_proj.weight, stop_gradient=False) + out_linear_weight = paddle.to_tensor( + self.out_proj.weight, stop_gradient=False) + ffn1_weight = paddle.to_tensor( + self.ffn1_proj.weight, stop_gradient=False) + ffn2_weight = paddle.to_tensor( + self.ffn2_proj.weight, stop_gradient=False) + + if self.bias_attr is False: + qkv_bias_tensor = None + out_linear_bias = None + else: + q_proj_bias = paddle.to_tensor( + self.q_proj.bias, stop_gradient=False) + k_proj_bias = paddle.to_tensor( + self.k_proj.bias, stop_gradient=False) + v_proj_bias = paddle.to_tensor( + self.v_proj.bias, stop_gradient=False) + qkv_bias = np.concatenate( + (q_proj_bias.numpy(), k_proj_bias.numpy(), v_proj_bias.numpy())) + qkv_bias = qkv_bias.reshape((3, self.num_heads, self.head_dim)) + qkv_bias_tensor = paddle.to_tensor(qkv_bias, stop_gradient=False) + out_linear_bias = paddle.to_tensor( + self.out_proj.bias, stop_gradient=False) + ffn1_bias = paddle.to_tensor( + self.ffn1_proj.bias, stop_gradient=False) + ffn2_bias = paddle.to_tensor( + self.ffn2_proj.bias, stop_gradient=False) + + ln_scale = paddle.to_tensor(self.norm.weight, stop_gradient=False) + ln_bias = paddle.to_tensor(self.norm.bias, stop_gradient=False) + ffn_ln_scale = paddle.to_tensor( + self.ffn_norm.weight, stop_gradient=False) + ffn_ln_bias = paddle.to_tensor(self.ffn_norm.bias, stop_gradient=False) + + q_proj_weight = q_proj_weight.numpy().transpose((1, 0)) + k_proj_weight = k_proj_weight.numpy().transpose((1, 0)) + v_proj_weight = v_proj_weight.numpy().transpose((1, 0)) + qkv_weight = np.concatenate( + (q_proj_weight, k_proj_weight, v_proj_weight)) + qkv_weight = qkv_weight.reshape( + (3, self.num_heads, self.head_dim, self.embed_dim)) + + x = paddle.to_tensor(self.query, stop_gradient=False) + cache_kvs, cache_kv = None, None + time_step = None + if self.has_cache_kv: + cache_kvs = [] + + max_seq_length = (self.cache_length + 128) // 128 * 128 + cache_kv = np.zeros( + [ + 2, self.batch_size, self.num_heads, max_seq_length, + self.head_dim + ], + dtype=self.x_type) + + elems = 4 + if self.x_type is np.float16: + elems = 8 + + assert self.head_dim % elems == 0 + v_elems = self.head_dim // elems + + # [B, num_head, 128, head_dim] + # cache_k_tmp = self.cache_kv[0, :] + # [B, num_head, 128, head_dim / 4, 4] + cache_k_tmp = self.cache_kv[0].reshape([ + self.batch_size, self.num_heads, self.cache_length, v_elems, + elems + ]) + # [B, num_head, head_dim / 4, 128, 4] + cache_k_tmp = cache_k_tmp.transpose([0, 1, 3, 2, 4]) + + cache_kv[0, :].reshape([ + self.batch_size, self.num_heads, v_elems, max_seq_length, elems + ])[:, :, :, :self.cache_length, :] = cache_k_tmp + + cache_kv[1, :, :, :self.cache_length, :] = self.cache_kv[1] + if self.gen_cache_kv: + assert self.query_length == self.cache_length + cache_kv[:] = 0 + else: + time_step = paddle.to_tensor( + [self.cache_length], dtype='int32', place=paddle.CPUPlace()) + if self.has_attn_mask: + attn_mask = paddle.to_tensor(self.attn_mask, stop_gradient=False) + else: + attn_mask = None + qkv_weight_tensor = paddle.to_tensor(qkv_weight, stop_gradient=False) + epsilon = 1e-05 + ln2_epsilon = 1e-05 + + if attn_mask is not None: + attn_mask = _convert_attention_mask(attn_mask, x.dtype) + + qkv_weights, qkv_biases = [], [] + out_weights, out_biases = [], [] + ln_scales, ln_biases = [], [] + ffn1_weights, ffn1_biases = [], [] + ffn2_weights, ffn2_biases = [], [] + ffn_ln_scales, ffn_ln_biases = [], [] + for i in range(self.layers): + qkv_weights.append(qkv_weight_tensor) + qkv_biases.append(qkv_bias_tensor) + out_weights.append(out_linear_weight) + out_biases.append(out_linear_bias) + ln_scales.append(ln_scale) + ln_biases.append(ln_bias) + ffn1_weights.append(ffn1_weight) + ffn1_biases.append(ffn1_bias) + ffn2_weights.append(ffn2_weight) + ffn2_biases.append(ffn2_bias) + ffn_ln_scales.append(ffn_ln_scale) + ffn_ln_biases.append(ffn_ln_bias) + if self.has_cache_kv: + cache_kvs.append( + paddle.to_tensor( + cache_kv, stop_gradient=False)) + + final_out = fused_multi_transformer( + x, + ln_scales, + ln_biases, + qkv_weights, + qkv_biases, + out_weights, + out_biases, + ffn_ln_scales, + ffn_ln_biases, + ffn1_weights, + ffn1_biases, + ffn2_weights, + ffn2_biases, + pre_layer_norm=self.pre_layer_norm, + epsilon=epsilon, + cache_kvs=cache_kvs, + time_step=time_step, + attn_mask=attn_mask, + dropout_rate=self.dropout_prob, + training=self.training) + + if self.has_cache_kv: + return final_out[0], final_out[1] + + return final_out + + def test_fused_multi_transformer_op(self): + final_out_ref = self.GetBaselineOut() + final_out = self.GetFusedMultiTransformerOut() + if self.has_cache_kv: + final_out, cache_kv_out = final_out + s = cache_kv_out[0].shape + bsz = s[1] + num_head = s[2] + max_seq_len = s[3] + head_dim = s[4] + elems = 8 if self.x_type is np.float16 else 4 + v_elems = head_dim // elems + + if self.debug: + print("cache_k out timestep=128") + print(cache_kv_out[0].reshape([ + 2, bsz, num_head, v_elems, max_seq_len, elems + ])[0, 0, 0, :, self.cache_length, :]) + + print("cache_v out timestep=128") + print(cache_kv_out[0][1, 0, 0, self.cache_length, :]) + + if self.gen_cache_kv: + final_out_ref, cache_kvs = final_out_ref + for i in range(self.layers): + cache_k_ref = cache_kvs[i][0] + cache_v_ref = cache_kvs[i][1] + + cache_k = cache_kv_out[i][0, :] + cache_k = cache_k.reshape( + [bsz, num_head, v_elems, max_seq_len, elems]) + cache_k = cache_k[:, :, :, :self.cache_length, :] + cache_k = cache_k.transpose([0, 1, 3, 2, 4]) + cache_k = cache_k.reshape( + [bsz, num_head, self.cache_length, head_dim]) + + cache_v = cache_kv_out[i][1, :, :, :self.cache_length, :] + + np.testing.assert_allclose( + cache_k_ref, cache_k, rtol=self.rtol, atol=self.atol) + np.testing.assert_allclose( + cache_v_ref, cache_v, rtol=self.rtol, atol=self.atol) + if i == 0: + break + + np.testing.assert_allclose( + final_out_ref, final_out, rtol=self.rtol, atol=self.atol) + + +class TestFusedMultiTransformerOpFp16(TestFusedMultiTransformerOp): + def config(self): + super().config() + self.x_type = np.float16 + self.layers = 3 # odd layers + + +class TestFusedMultiTransformerOpCacheKV(TestFusedMultiTransformerOp): + def config(self): + super().config() + self.has_cache_kv = True + self.query_length = 1 + self.key_length, self.value_length = 1, 1 + self.layers = 3 # odd layers + + +class TestFusedMultiTransformerOpCacheKVFp16(TestFusedMultiTransformerOp): + def config(self): + super().config() + self.has_cache_kv = True + self.query_length = 1 + self.key_length, self.value_length = 1, 1 + self.x_type = np.float16 + + +class TestFusedMultiTransformerOpGenCacheKV(TestFusedMultiTransformerOp): + def config(self): + super().config() + self.has_cache_kv = True + self.gen_cache_kv = True + + +class TestFusedMultiTransformerOpGenCacheKVFp16(TestFusedMultiTransformerOp): + def config(self): + super().config() + self.has_cache_kv = True + self.gen_cache_kv = True + self.x_type = np.float16 + self.layers = 3 # odd layers + + +if __name__ == "__main__": + unittest.main() diff --git a/python/paddle/fluid/tests/unittests/test_label_smooth_functional.py b/python/paddle/fluid/tests/unittests/test_label_smooth_functional.py index 54f5e64fda4b6..83c8ced79b1e8 100644 --- a/python/paddle/fluid/tests/unittests/test_label_smooth_functional.py +++ b/python/paddle/fluid/tests/unittests/test_label_smooth_functional.py @@ -19,6 +19,8 @@ import paddle.nn.functional as F import paddle.fluid.initializer as I import unittest +from paddle.fluid.framework import _enable_legacy_dygraph +_enable_legacy_dygraph() class LabelSmoothTestCase(unittest.TestCase): diff --git a/python/paddle/fluid/tests/unittests/test_layout_autotune.py b/python/paddle/fluid/tests/unittests/test_layout_autotune.py index c71ff4381028d..a1440f8587ab6 100644 --- a/python/paddle/fluid/tests/unittests/test_layout_autotune.py +++ b/python/paddle/fluid/tests/unittests/test_layout_autotune.py @@ -16,6 +16,10 @@ import unittest import numpy import paddle.nn.functional as F +import tempfile +import warnings +import json +import os class SimpleNet(paddle.nn.Layer): @@ -41,10 +45,18 @@ def forward(self, image): class LayoutAutoTune(unittest.TestCase): def use_autoune(self): if paddle.is_compiled_with_cuda(): - paddle.fluid.core.enable_layout_autotune() + paddle.incubate.autotune.set_config( + config={"layout": { + "enable": True + }}) return paddle.fluid.core.use_layout_autotune() else: - paddle.fluid.core.disable_layout_autotune() + config = {"layout": {"enable": False}} + tfile = tempfile.NamedTemporaryFile(mode="w+", delete=False) + json.dump(config, tfile) + tfile.close() + paddle.incubate.autotune.set_config(tfile.name) + os.remove(tfile.name) return paddle.fluid.core.use_layout_autotune() def train(self, data_format): @@ -103,7 +115,6 @@ def test_transpose_op_transposer(self): def test_flatten_op_transposer(self): if not self.use_autoune(): return - paddle.fluid.core.enable_layout_autotune() conv = paddle.nn.Conv2D(3, 8, (3, 3)) flatten = paddle.nn.Flatten(start_axis=1, stop_axis=2) data = paddle.rand([1, 3, 16, 14]) @@ -119,5 +130,20 @@ def test_flatten_op_transposer(self): self.assertEqual(out.shape, [1, 112, 12]) +class TestAutoTuneAPI(unittest.TestCase): + def test_set_config_warnings(self): + with warnings.catch_warnings(record=True) as w: + config = {"layout": {"enable": 1}} + # On linux, we can open the file again to read the content + # without closing the file, but on windows system, there is + # no permission to open it again without closing it. + tfile = tempfile.NamedTemporaryFile(mode="w+", delete=False) + json.dump(config, tfile) + tfile.close() + paddle.incubate.autotune.set_config(tfile.name) + os.remove(tfile.name) + self.assertTrue(len(w) == 1) + + if __name__ == '__main__': unittest.main() diff --git a/python/paddle/fluid/tests/unittests/test_lbfgs.py b/python/paddle/fluid/tests/unittests/test_lbfgs.py index bb3818747601f..2cad4822b28b1 100644 --- a/python/paddle/fluid/tests/unittests/test_lbfgs.py +++ b/python/paddle/fluid/tests/unittests/test_lbfgs.py @@ -21,6 +21,9 @@ from paddle.incubate.optimizer.functional.lbfgs import minimize_lbfgs +from paddle.fluid.framework import _enable_legacy_dygraph +_enable_legacy_dygraph() + np.random.seed(123) diff --git a/python/paddle/fluid/tests/unittests/test_multinomial_op.py b/python/paddle/fluid/tests/unittests/test_multinomial_op.py index ecde527523d3d..4dfc881d7723f 100644 --- a/python/paddle/fluid/tests/unittests/test_multinomial_op.py +++ b/python/paddle/fluid/tests/unittests/test_multinomial_op.py @@ -216,6 +216,14 @@ def test_dim_less_than_1(): self.assertRaises(ValueError, test_dim_less_than_1) + with self.assertRaises(ValueError): + y = paddle.multinomial(paddle.to_tensor([1., 2., -3.])) + + with self.assertRaises(ValueError): + prob = paddle.rand([20, 1000]) + prob[1:0] = 0 + y = paddle.multinomial(prob) + class TestRandomValue(unittest.TestCase): def test_fixed_random_number(self): diff --git a/python/paddle/fluid/tests/unittests/test_nan_inf.py b/python/paddle/fluid/tests/unittests/test_nan_inf.py index 84559048a2b8a..9b11f6711afc1 100644 --- a/python/paddle/fluid/tests/unittests/test_nan_inf.py +++ b/python/paddle/fluid/tests/unittests/test_nan_inf.py @@ -20,6 +20,8 @@ import sys import subprocess import paddle +from paddle.fluid.framework import _enable_legacy_dygraph +_enable_legacy_dygraph() paddle.enable_static() diff --git a/python/paddle/fluid/tests/unittests/test_newprofiler.py b/python/paddle/fluid/tests/unittests/test_newprofiler.py index ae804f82b90f7..53ade0dfb79c1 100755 --- a/python/paddle/fluid/tests/unittests/test_newprofiler.py +++ b/python/paddle/fluid/tests/unittests/test_newprofiler.py @@ -134,6 +134,42 @@ def my_sheduler1(num_step): prof.export(path='./test_profiler_pb.pb', format='pb') prof.summary() result = profiler.utils.load_profiler_result('./test_profiler_pb.pb') + prof = None + dataset = RandomDataset(10 * 4) + simple_net = SimpleNet() + opt = paddle.optimizer.SGD(learning_rate=1e-3, + parameters=simple_net.parameters()) + loader = DataLoader( + dataset, batch_size=4, shuffle=True, drop_last=True, num_workers=2) + prof = profiler.Profiler(on_trace_ready=lambda prof: None) + prof.start() + for i, (image, label) in enumerate(loader()): + out = simple_net(image) + loss = F.cross_entropy(out, label) + avg_loss = paddle.mean(loss) + avg_loss.backward() + opt.minimize(avg_loss) + simple_net.clear_gradients() + prof.step() + prof.stop() + prof.summary() + prof = None + dataset = RandomDataset(10 * 4) + simple_net = SimpleNet() + loader = DataLoader(dataset, batch_size=4, shuffle=True, drop_last=True) + opt = paddle.optimizer.Adam( + learning_rate=1e-3, parameters=simple_net.parameters()) + prof = profiler.Profiler(on_trace_ready=lambda prof: None) + prof.start() + for i, (image, label) in enumerate(loader()): + out = simple_net(image) + loss = F.cross_entropy(out, label) + avg_loss = paddle.mean(loss) + avg_loss.backward() + opt.step() + simple_net.clear_gradients() + prof.step() + prof.stop() class TestNvprof(unittest.TestCase): diff --git a/python/paddle/fluid/tests/unittests/test_nn_functional_embedding_dygraph.py b/python/paddle/fluid/tests/unittests/test_nn_functional_embedding_dygraph.py index acff7daadeb33..0b5493e21705f 100644 --- a/python/paddle/fluid/tests/unittests/test_nn_functional_embedding_dygraph.py +++ b/python/paddle/fluid/tests/unittests/test_nn_functional_embedding_dygraph.py @@ -19,12 +19,13 @@ import paddle import paddle.nn as nn import numpy as np +from paddle.fluid.framework import _test_eager_guard paddle.disable_static() class EmbeddingDygraph(unittest.TestCase): - def test_1(self): + def func_1(self): x_data = np.arange(3, 6).reshape((3, 1)).astype(np.int64) paddle.disable_static(paddle.CPUPlace()) x = paddle.to_tensor(x_data, stop_gradient=False) @@ -42,7 +43,12 @@ def test_1(self): out.backward() adam.step() - def test_2(self): + def test_1(self): + with _test_eager_guard(): + self.func_1() + self.func_1() + + def func_2(self): x_data = np.arange(3, 6).reshape((3, 1)).astype(np.int64) y_data = np.arange(6, 12).reshape((3, 2)).astype(np.float32) paddle.disable_static(paddle.CPUPlace()) @@ -58,6 +64,11 @@ def test_2(self): with self.assertRaises(ValueError): embedding = paddle.nn.Embedding(10, -3, sparse=True) + def test_2(self): + with _test_eager_guard(): + self.func_2() + self.func_2() + if __name__ == '__main__': unittest.main() diff --git a/python/paddle/fluid/tests/unittests/test_pixel_unshuffle.py b/python/paddle/fluid/tests/unittests/test_pixel_unshuffle.py new file mode 100644 index 0000000000000..768a9e307c91e --- /dev/null +++ b/python/paddle/fluid/tests/unittests/test_pixel_unshuffle.py @@ -0,0 +1,294 @@ +# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import unittest +import numpy as np + +from op_test import OpTest +import paddle +import paddle.nn.functional as F +import paddle.fluid.core as core +import paddle.fluid as fluid + + +def pixel_unshuffle_np(x, down_factor, data_format="NCHW"): + '''Numpy implementation of pixel unshuffle''' + + if data_format == "NCHW": + n, c, h, w = x.shape + new_shape = (n, c, h // down_factor, down_factor, w // down_factor, + down_factor) + npresult = np.reshape(x, new_shape) + npresult = npresult.transpose(0, 1, 3, 5, 2, 4) + oshape = [ + n, c * down_factor * down_factor, h // down_factor, w // down_factor + ] + npresult = np.reshape(npresult, oshape) + return npresult + else: + n, h, w, c = x.shape + new_shape = (n, h // down_factor, down_factor, w // down_factor, + down_factor, c) + npresult = np.reshape(x, new_shape) + npresult = npresult.transpose(0, 1, 3, 5, 2, 4) + oshape = [ + n, h // down_factor, w // down_factor, c * down_factor * down_factor + ] + npresult = np.reshape(npresult, oshape) + return npresult + + +class TestPixelUnshuffleOp(OpTest): + '''TestPixelUnshuffleOp''' + + def setUp(self): + '''setUp''' + + self.op_type = "pixel_unshuffle" + self.init_data_format() + n, c, h, w = 2, 1, 12, 12 + + if self.format == "NCHW": + shape = [n, c, h, w] + if self.format == "NHWC": + shape = [n, h, w, c] + + down_factor = 3 + + x = np.random.random(shape).astype("float64") + npresult = pixel_unshuffle_np(x, down_factor, self.format) + + self.inputs = {"X": x} + self.outputs = {"Out": npresult} + self.attrs = { + "downscale_factor": down_factor, + "data_format": self.format + } + + def init_data_format(self): + '''init_data_format''' + + self.format = "NCHW" + + def test_check_output(self): + '''test_check_output''' + + self.check_output() + + def test_check_grad(self): + '''test_check_grad''' + + self.check_grad(["X"], "Out") + + +class TestChannelLast(TestPixelUnshuffleOp): + '''TestChannelLast''' + + def init_data_format(self): + '''init_data_format''' + + self.format = "NHWC" + + +class TestPixelUnshuffleAPI(unittest.TestCase): + '''TestPixelUnshuffleAPI''' + + def setUp(self): + '''setUp''' + + self.x_1_np = np.random.random([2, 1, 12, 12]).astype("float64") + self.x_2_np = np.random.random([2, 12, 12, 1]).astype("float64") + self.out_1_np = pixel_unshuffle_np(self.x_1_np, 3) + self.out_2_np = pixel_unshuffle_np(self.x_2_np, 3, "NHWC") + + def test_static_graph_functional(self): + '''test_static_graph_functional''' + + for use_cuda in ([False, True] + if core.is_compiled_with_cuda() else [False]): + place = paddle.CUDAPlace(0) if use_cuda else paddle.CPUPlace() + + paddle.enable_static() + x_1 = paddle.fluid.data( + name="x", shape=[2, 1, 12, 12], dtype="float64") + x_2 = paddle.fluid.data( + name="x2", shape=[2, 12, 12, 1], dtype="float64") + out_1 = F.pixel_unshuffle(x_1, 3) + out_2 = F.pixel_unshuffle(x_2, 3, "NHWC") + + exe = paddle.static.Executor(place=place) + res_1 = exe.run(fluid.default_main_program(), + feed={"x": self.x_1_np}, + fetch_list=out_1, + use_prune=True) + + res_2 = exe.run(fluid.default_main_program(), + feed={"x2": self.x_2_np}, + fetch_list=out_2, + use_prune=True) + + assert np.allclose(res_1, self.out_1_np) + assert np.allclose(res_2, self.out_2_np) + + # same test between layer and functional in this op. + def test_static_graph_layer(self): + '''test_static_graph_layer''' + + for use_cuda in ([False, True] + if core.is_compiled_with_cuda() else [False]): + place = paddle.CUDAPlace(0) if use_cuda else paddle.CPUPlace() + + paddle.enable_static() + x_1 = paddle.fluid.data( + name="x", shape=[2, 1, 12, 12], dtype="float64") + x_2 = paddle.fluid.data( + name="x2", shape=[2, 12, 12, 1], dtype="float64") + # init instance + ps_1 = paddle.nn.PixelUnshuffle(3) + ps_2 = paddle.nn.PixelUnshuffle(3, "NHWC") + out_1 = ps_1(x_1) + out_2 = ps_2(x_2) + out_1_np = pixel_unshuffle_np(self.x_1_np, 3) + out_2_np = pixel_unshuffle_np(self.x_2_np, 3, "NHWC") + + exe = paddle.static.Executor(place=place) + res_1 = exe.run(fluid.default_main_program(), + feed={"x": self.x_1_np}, + fetch_list=out_1, + use_prune=True) + + res_2 = exe.run(fluid.default_main_program(), + feed={"x2": self.x_2_np}, + fetch_list=out_2, + use_prune=True) + + assert np.allclose(res_1, out_1_np) + assert np.allclose(res_2, out_2_np) + + def run_dygraph(self, down_factor, data_format): + '''run_dygraph''' + + n, c, h, w = 2, 1, 12, 12 + + if data_format == "NCHW": + shape = [n, c, h, w] + if data_format == "NHWC": + shape = [n, h, w, c] + + x = np.random.random(shape).astype("float64") + + npresult = pixel_unshuffle_np(x, down_factor, data_format) + + for use_cuda in ([False, True] + if core.is_compiled_with_cuda() else [False]): + place = paddle.CUDAPlace(0) if use_cuda else paddle.CPUPlace() + + paddle.disable_static(place=place) + + pixel_unshuffle = paddle.nn.PixelUnshuffle( + down_factor, data_format=data_format) + result = pixel_unshuffle(paddle.to_tensor(x)) + + self.assertTrue(np.allclose(result.numpy(), npresult)) + + result_functional = F.pixel_unshuffle( + paddle.to_tensor(x), 3, data_format) + self.assertTrue(np.allclose(result_functional.numpy(), npresult)) + + pixel_unshuffle_str = 'downscale_factor={}'.format(down_factor) + if data_format != 'NCHW': + pixel_unshuffle_str += ', data_format={}'.format(data_format) + self.assertEqual(pixel_unshuffle.extra_repr(), pixel_unshuffle_str) + + def test_dygraph1(self): + '''test_dygraph1''' + + self.run_dygraph(3, "NCHW") + + def test_dygraph2(self): + '''test_dygraph2''' + + self.run_dygraph(3, "NHWC") + + +class TestPixelUnshuffleError(unittest.TestCase): + '''TestPixelUnshuffleError''' + + def test_error_functional(self): + '''test_error_functional''' + + def error_input(): + with paddle.fluid.dygraph.guard(): + x = np.random.random([4, 12, 12]).astype("float64") + pixel_unshuffle = F.pixel_unshuffle(paddle.to_tensor(x), 2) + + self.assertRaises(ValueError, error_input) + + def error_downscale_factor_1(): + with paddle.fluid.dygraph.guard(): + x = np.random.random([2, 1, 12, 12]).astype("float64") + pixel_unshuffle = F.pixel_unshuffle(paddle.to_tensor(x), 3.33) + + self.assertRaises(TypeError, error_downscale_factor_1) + + def error_downscale_factor_2(): + with paddle.fluid.dygraph.guard(): + x = np.random.random([2, 1, 12, 12]).astype("float64") + pixel_unshuffle = F.pixel_unshuffle(paddle.to_tensor(x), -1) + + self.assertRaises(ValueError, error_downscale_factor_2) + + def error_data_format(): + with paddle.fluid.dygraph.guard(): + x = np.random.random([2, 1, 12, 12]).astype("float64") + pixel_unshuffle = F.pixel_unshuffle( + paddle.to_tensor(x), 3, "WOW") + + self.assertRaises(ValueError, error_data_format) + + def test_error_layer(self): + '''test_error_layer''' + + def error_input_layer(): + with paddle.fluid.dygraph.guard(): + x = np.random.random([4, 12, 12]).astype("float64") + ps = paddle.nn.PixelUnshuffle(2) + ps(paddle.to_tensor(x)) + + self.assertRaises(ValueError, error_input_layer) + + def error_downscale_factor_layer_1(): + with paddle.fluid.dygraph.guard(): + x = np.random.random([2, 1, 12, 12]).astype("float64") + ps = paddle.nn.PixelUnshuffle(3.33) + + self.assertRaises(TypeError, error_downscale_factor_layer_1) + + def error_downscale_factor_layer_2(): + with paddle.fluid.dygraph.guard(): + x = np.random.random([2, 1, 12, 12]).astype("float64") + ps = paddle.nn.PixelUnshuffle(-1) + + self.assertRaises(ValueError, error_downscale_factor_layer_2) + + def error_data_format_layer(): + with paddle.fluid.dygraph.guard(): + x = np.random.random([2, 1, 12, 12]).astype("float64") + ps = paddle.nn.PixelUnshuffle(3, "MEOW") + + self.assertRaises(ValueError, error_data_format_layer) + + +if __name__ == "__main__": + unittest.main() diff --git a/python/paddle/fluid/tests/unittests/test_qr_op.py b/python/paddle/fluid/tests/unittests/test_qr_op.py index 4be46837a67ae..ecf65d16d3431 100644 --- a/python/paddle/fluid/tests/unittests/test_qr_op.py +++ b/python/paddle/fluid/tests/unittests/test_qr_op.py @@ -27,7 +27,7 @@ class TestQrOp(OpTest): def setUp(self): paddle.enable_static() - np.random.seed(4) + np.random.seed(7) self.op_type = "qr" a, q, r = self.get_input_and_output() self.inputs = {"X": a} @@ -74,7 +74,8 @@ def test_check_output(self): self.check_output() def test_check_grad_normal(self): - self.check_grad(['X'], ['Q', 'R']) + self.check_grad( + ['X'], ['Q', 'R'], numeric_grad_delta=1e-5, max_relative_error=1e-6) class TestQrOpCase1(TestQrOp): @@ -116,6 +117,7 @@ def get_shape(self): class TestQrAPI(unittest.TestCase): def test_dygraph(self): paddle.disable_static() + np.random.seed(7) def run_qr_dygraph(shape, mode, dtype): if dtype == "float32": @@ -180,6 +182,7 @@ def run_qr_dygraph(shape, mode, dtype): def test_static(self): paddle.enable_static() + np.random.seed(7) def run_qr_static(shape, mode, dtype): if dtype == "float32": diff --git a/python/paddle/fluid/tests/unittests/test_sparse_conv_op.py b/python/paddle/fluid/tests/unittests/test_sparse_conv_op.py index 42f628c8fb1fd..1677051ee9db4 100644 --- a/python/paddle/fluid/tests/unittests/test_sparse_conv_op.py +++ b/python/paddle/fluid/tests/unittests/test_sparse_conv_op.py @@ -31,19 +31,21 @@ def test_conv3d(self): paddings = [0, 0, 0] strides = [1, 1, 1] dilations = [1, 1, 1] + bias = [1] indices = [[0, 0, 0, 0], [0, 0, 0, 0], [0, 0, 1, 2], [1, 3, 2, 3]] values = [1, 2, 3, 4] indices = paddle.to_tensor(indices, dtype='int32') values = paddle.to_tensor(values, dtype='float32') dense_shape = [1, 1, 3, 4, 1] - correct_out_values = [[4], [10]] + correct_out_values = [[5], [11]] sparse_input = core.eager.sparse_coo_tensor(indices, values, dense_shape, False) out = paddle.sparse.functional.conv3d( sparse_input, dense_kernel, - bias=None, + bias=paddle.to_tensor( + bias, dtype='float32'), stride=strides, padding=paddings, dilation=dilations, diff --git a/python/paddle/fluid/tests/unittests/test_sparse_norm_op.py b/python/paddle/fluid/tests/unittests/test_sparse_norm_op.py new file mode 100644 index 0000000000000..3c3085ec8be69 --- /dev/null +++ b/python/paddle/fluid/tests/unittests/test_sparse_norm_op.py @@ -0,0 +1,87 @@ +# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import print_function +import unittest +import numpy as np +import paddle +from paddle.fluid.framework import _test_eager_guard +import copy + + +class TestSparseBatchNorm(unittest.TestCase): + def test(self): + with _test_eager_guard(): + paddle.seed(0) + channels = 4 + shape = [2, 3, 6, 6, channels] + #there is no zero in dense_x + dense_x = paddle.randn(shape) + dense_x.stop_gradient = False + + batch_norm = paddle.nn.BatchNorm3D(channels, data_format="NDHWC") + dense_y = batch_norm(dense_x) + dense_y.backward(dense_y) + + sparse_dim = 4 + dense_x2 = copy.deepcopy(dense_x) + dense_x2.stop_gradient = False + sparse_x = dense_x2.to_sparse_coo(sparse_dim) + sparse_batch_norm = paddle.sparse.BatchNorm(channels) + # set same params + sparse_batch_norm._mean.set_value(batch_norm._mean) + sparse_batch_norm._variance.set_value(batch_norm._variance) + sparse_batch_norm.weight.set_value(batch_norm.weight) + + sparse_y = sparse_batch_norm(sparse_x) + # compare the result with dense batch_norm + assert np.allclose( + dense_y.flatten().numpy(), + sparse_y.values().flatten().numpy(), + atol=1e-5, + rtol=1e-5) + + # test backward + sparse_y.backward(sparse_y) + assert np.allclose( + dense_x.grad.flatten().numpy(), + sparse_x.grad.values().flatten().numpy(), + atol=1e-5, + rtol=1e-5) + + def test_error_layout(self): + with _test_eager_guard(): + with self.assertRaises(ValueError): + shape = [2, 3, 6, 6, 3] + x = paddle.randn(shape) + sparse_x = x.to_sparse_coo(4) + sparse_batch_norm = paddle.sparse.BatchNorm( + 3, data_format='NCDHW') + sparse_batch_norm(sparse_x) + + def test2(self): + with _test_eager_guard(): + paddle.seed(123) + channels = 3 + x_data = paddle.randn((1, 6, 6, 6, channels)).astype('float32') + dense_x = paddle.to_tensor(x_data) + sparse_x = dense_x.to_sparse_coo(4) + batch_norm = paddle.sparse.BatchNorm(channels) + batch_norm_out = batch_norm(sparse_x) + print(batch_norm_out.shape) + # [1, 6, 6, 6, 3] + + +if __name__ == "__main__": + unittest.main() diff --git a/python/paddle/fluid/tests/unittests/test_sparse_pooling_op.py b/python/paddle/fluid/tests/unittests/test_sparse_pooling_op.py new file mode 100644 index 0000000000000..a1a3849f7191b --- /dev/null +++ b/python/paddle/fluid/tests/unittests/test_sparse_pooling_op.py @@ -0,0 +1,112 @@ +# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import print_function +import unittest +import numpy as np +import paddle +import paddle.fluid.core as core +from paddle import _C_ops +from paddle.fluid.framework import _test_eager_guard + + +class TestMaxPool3DFunc(unittest.TestCase): + def setInput(self): + paddle.seed(0) + self.dense_x = paddle.randn((1, 4, 4, 4, 4)) + + def setKernelSize(self): + self.kernel_sizes = [3, 3, 3] + + def setStride(self): + self.strides = [1, 1, 1] + + def setPadding(self): + self.paddings = [0, 0, 0] + + def setUp(self): + self.setInput() + self.setKernelSize() + self.setStride() + self.setPadding() + + def test(self): + with _test_eager_guard(): + self.setUp() + sparse_x = self.dense_x.to_sparse_coo(4) + out = paddle.sparse.functional.max_pool3d( + sparse_x, + self.kernel_sizes, + stride=self.strides, + padding=self.paddings) + out = out.to_dense() + + dense_out = paddle.nn.functional.max_pool3d( + self.dense_x, + self.kernel_sizes, + stride=self.strides, + padding=self.paddings, + data_format='NDHWC') + #compare with dense + assert np.allclose(dense_out.flatten().numpy(), + out.flatten().numpy()) + + +class TestStride(TestMaxPool3DFunc): + def setStride(self): + self.strides = 1 + + +class TestPadding(TestMaxPool3DFunc): + def setPadding(self): + self.paddings = 1 + + def setInput(self): + self.dense_x = paddle.randn((1, 5, 6, 8, 3)) + + +class TestKernelSize(TestMaxPool3DFunc): + def setKernelSize(self): + self.kernel_sizes = [5, 5, 5] + + def setInput(self): + paddle.seed(0) + self.dense_x = paddle.randn((1, 6, 9, 6, 3)) + + +class TestInput(TestMaxPool3DFunc): + def setInput(self): + paddle.seed(0) + self.dense_x = paddle.randn((2, 6, 7, 9, 3)) + dropout = paddle.nn.Dropout(0.8) + self.dense_x = dropout(self.dense_x) + + +class TestMaxPool3DAPI(unittest.TestCase): + def test(self): + with _test_eager_guard(): + dense_x = paddle.randn((2, 3, 6, 6, 3)) + sparse_x = dense_x.to_sparse_coo(4) + max_pool3d = paddle.sparse.MaxPool3D( + kernel_size=3, data_format='NDHWC') + out = max_pool3d(sparse_x) + out = out.to_dense() + + dense_out = paddle.nn.functional.max_pool3d( + dense_x, 3, data_format='NDHWC') + assert np.allclose(dense_out.numpy(), out.numpy()) + + +if __name__ == "__main__": + unittest.main() diff --git a/python/paddle/fluid/tests/unittests/test_sparse_utils_op.py b/python/paddle/fluid/tests/unittests/test_sparse_utils_op.py index c87626a10c631..80820c0f2d837 100644 --- a/python/paddle/fluid/tests/unittests/test_sparse_utils_op.py +++ b/python/paddle/fluid/tests/unittests/test_sparse_utils_op.py @@ -208,6 +208,20 @@ def test_coo_values_grad(self): # test coo_values_grad values_tensor.backward(paddle.to_tensor(out_grad)) assert np.array_equal(out_grad, sparse_x.grad.values().numpy()) + indices = [[0, 0, 1, 2, 2], [1, 3, 2, 0, 1]] + values = [[1.0, 1.0], [2.0, 2.0], [3.0, 3.0], [4.0, 4.0], + [5.0, 5.0]] + sparse_x = paddle.sparse.sparse_coo_tensor( + paddle.to_tensor(indices), + paddle.to_tensor(values), + shape=[3, 4, 2], + stop_gradient=False) + values_tensor = sparse_x.values() + out_grad = [[2.0, 2.0], [3.0, 3.0], [5.0, 5.0], [8.0, 8.0], + [9.0, 9.0]] + # test coo_values_grad + values_tensor.backward(paddle.to_tensor(out_grad)) + assert np.array_equal(out_grad, sparse_x.grad.values().numpy()) def test_sparse_coo_tensor_grad(self): with _test_eager_guard(): @@ -233,6 +247,21 @@ def test_sparse_coo_tensor_grad(self): assert np.array_equal(correct_values_grad, values.grad.numpy()) + # test the non-zero values is a vector + values = [[1, 1], [2, 2]] + values = paddle.to_tensor( + values, dtype='float32', stop_gradient=False) + sparse_x = paddle.sparse.sparse_coo_tensor( + indices, values, shape=[2, 2, 2], stop_gradient=False) + grad_values = [[2, 2], [3, 3]] + grad_values = paddle.to_tensor(grad_values, dtype='float32') + sparse_out_grad = paddle.sparse.sparse_coo_tensor( + grad_indices, grad_values, shape=[2, 2, 2]) + sparse_x.backward(sparse_out_grad) + correct_values_grad = [[0, 0], [3, 3]] + assert np.array_equal(correct_values_grad, + values.grad.numpy()) + def test_sparse_coo_tensor_sorted(self): with _test_eager_guard(): for device in devices: @@ -252,6 +281,16 @@ def test_sparse_coo_tensor_sorted(self): assert np.array_equal(values_sorted, sparse_x.values().numpy()) + # test the non-zero values is a vector + values = [[1.0, 1.0], [2.0, 2.0], [3.0, 3.0]] + values = paddle.to_tensor(values, dtype='float32') + sparse_x = paddle.sparse.sparse_coo_tensor(indices, values) + values_sorted = [[5.0, 5.0], [1.0, 1.0]] + assert np.array_equal(indices_sorted, + sparse_x.indices().numpy()) + assert np.array_equal(values_sorted, + sparse_x.values().numpy()) + class TestCooError(unittest.TestCase): def test_small_shape(self): diff --git a/python/paddle/fluid/tests/unittests/test_static_model_parallel_fused_multi_transformer.py b/python/paddle/fluid/tests/unittests/test_static_model_parallel_fused_multi_transformer.py new file mode 100644 index 0000000000000..5475fd4a10a13 --- /dev/null +++ b/python/paddle/fluid/tests/unittests/test_static_model_parallel_fused_multi_transformer.py @@ -0,0 +1,45 @@ +# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import print_function +import unittest +from test_dist_base import TestDistBase + +import os +import paddle + +paddle.enable_static() +flag_name = os.path.splitext(__file__)[0] + + +class TestStaticModelParallel(TestDistBase): + def _setup_config(self): + self._sync_mode = True + self._use_reduce = False + self._use_reader_alloc = False + self._nccl_comm_num = 1 + self._pipeline_mode = True + + def test_dist_static_model_parallel_fused_multi_transformer(self): + import paddle.fluid as fluid + if fluid.core.is_compiled_with_cuda(): + self.check_with_place( + "static_model_parallel_fused_multi_transformer.py", + delta=1e-5, + check_error_log=True, + log_name=flag_name) + + +if __name__ == '__main__': + unittest.main() diff --git a/python/paddle/fluid/tests/unittests/test_switch_autotune.py b/python/paddle/fluid/tests/unittests/test_switch_autotune.py index 1775272aac69d..0049a922b9166 100644 --- a/python/paddle/fluid/tests/unittests/test_switch_autotune.py +++ b/python/paddle/fluid/tests/unittests/test_switch_autotune.py @@ -15,6 +15,10 @@ import paddle import unittest import numpy as np +import tempfile +import warnings +import json +import os class SimpleNet(paddle.nn.Layer): @@ -73,10 +77,13 @@ def get_expected_res(self, step_id, enable_autotune): return expected_res def test_autotune(self): - paddle.fluid.core.disable_autotune() + paddle.incubate.autotune.set_config( + config={"kernel": { + "enable": False + }}) self.assertEqual(self.get_flags("FLAGS_use_autotune"), False) - paddle.fluid.core.enable_autotune() + paddle.incubate.autotune.set_config(config={"kernel": {"enable": True}}) self.assertEqual(self.get_flags("FLAGS_use_autotune"), True) def check_status(self, expected_res): @@ -93,10 +100,16 @@ class TestDygraphAutoTuneStatus(TestAutoTune): def run_program(self, enable_autotune): self.set_flags(enable_autotune) if enable_autotune: - paddle.fluid.core.enable_autotune() + paddle.incubate.autotune.set_config( + config={"kernel": { + "enable": True, + "tuning_range": [1, 2] + }}) else: - paddle.fluid.core.disable_autotune() - paddle.fluid.core.set_autotune_range(1, 2) + paddle.incubate.autotune.set_config( + config={"kernel": { + "enable": False + }}) x_var = paddle.uniform((1, 1, 8, 8), dtype='float32', min=-1., max=1.) net = SimpleNet() for i in range(3): @@ -141,10 +154,18 @@ def run_program(self, enable_autotune): self.set_flags(enable_autotune) if enable_autotune: - paddle.fluid.core.enable_autotune() + config = {"kernel": {"enable": True, "tuning_range": [1, 2]}} + tfile = tempfile.NamedTemporaryFile(mode="w+", delete=False) + json.dump(config, tfile) + tfile.close() + paddle.incubate.autotune.set_config(tfile.name) + os.remove(tfile.name) else: - paddle.fluid.core.disable_autotune() - paddle.fluid.core.set_autotune_range(1, 2) + paddle.incubate.autotune.set_config( + config={"kernel": { + "enable": False, + "tuning_range": [1, 2] + }}) for i in range(3): exe.run(program=main_program, feed={'X': x}, fetch_list=[loss]) @@ -166,5 +187,22 @@ def test_disable_autotune(self): self.func_disable_autotune() +class TestAutoTuneAPI(unittest.TestCase): + def test_set_config_warnings(self): + with warnings.catch_warnings(record=True) as w: + config = {"kernel": {"enable": 1, "tuning_range": 1}} + tfile = tempfile.NamedTemporaryFile(mode="w+", delete=False) + json.dump(config, tfile) + tfile.close() + paddle.incubate.autotune.set_config(tfile.name) + os.remove(tfile.name) + self.assertTrue(len(w) == 2) + + def test_set_config_attr(self): + paddle.incubate.autotune.set_config(config=None) + self.assertEqual( + paddle.get_flags("FLAGS_use_autotune")["FLAGS_use_autotune"], True) + + if __name__ == '__main__': unittest.main() diff --git a/python/paddle/fluid/tests/unittests/test_tensor_register_hook.py b/python/paddle/fluid/tests/unittests/test_tensor_register_hook.py index a5ca53108fc59..e7f85f0451a17 100644 --- a/python/paddle/fluid/tests/unittests/test_tensor_register_hook.py +++ b/python/paddle/fluid/tests/unittests/test_tensor_register_hook.py @@ -462,11 +462,9 @@ def double_print_hook(grad): x.register_hook(double_print_hook) y = x * x - fluid.set_flags({'FLAGS_retain_grad_for_all_tensor': False}) # Since y = x * x, dx = 2 * x dx = paddle.grad( outputs=[y], inputs=[x], create_graph=True, retain_graph=True)[0] - fluid.set_flags({'FLAGS_retain_grad_for_all_tensor': True}) z = y + dx self.assertTrue(x.grad is None) diff --git a/python/paddle/fluid/tests/unittests/test_tensor_scalar_type_promotion_dynamic.py b/python/paddle/fluid/tests/unittests/test_tensor_scalar_type_promotion_dynamic.py index 5f2dfbdd99e16..774d40a17c66d 100644 --- a/python/paddle/fluid/tests/unittests/test_tensor_scalar_type_promotion_dynamic.py +++ b/python/paddle/fluid/tests/unittests/test_tensor_scalar_type_promotion_dynamic.py @@ -18,6 +18,7 @@ import numpy as np import paddle +from paddle.fluid.framework import _test_eager_guard # Support types are ref from `paddle.tensor.math` # - Related paddle dtypes: @@ -50,7 +51,7 @@ def check_operation(self, a, b, c, op): self.assertEqual(c_rlt.dtype, c.dtype) self.assertTrue(np.array_equal(c_rlt.numpy(), c.numpy())) - def test_tensor_add_scalar(self): + def func_tensor_add_scalar(self): # tensor(int64) + scalar(int) a = paddle.ones([2, 2, 2], dtype='int64') b = 1 @@ -81,7 +82,12 @@ def test_tensor_add_scalar(self): c = paddle.full([2, 2, 2], 2.5, dtype="float32") self.check_operation(a, b, c, '+') - def test_tensor_sub_scalar(self): + def test_tensor_add_scalar(self): + with _test_eager_guard(): + self.func_tensor_add_scalar() + self.func_tensor_add_scalar() + + def func_tensor_sub_scalar(self): # tensor(int64) - scalar(int) a = paddle.ones([2, 2, 2], dtype='int64') b = 1 @@ -112,7 +118,12 @@ def test_tensor_sub_scalar(self): c = paddle.full([2, 2, 2], 0.5, dtype="float32") self.check_operation(a, b, c, '-') - def test_scalar_sub_tensor(self): + def test_tensor_sub_scalar(self): + with _test_eager_guard(): + self.func_tensor_sub_scalar() + self.func_tensor_sub_scalar() + + def func_scalar_sub_tensor(self): # scalar(int) - tensor(int64) a = 1 b = paddle.ones([2, 2, 2], dtype='int64') @@ -143,7 +154,12 @@ def test_scalar_sub_tensor(self): c = paddle.full([2, 2, 2], -0.5, dtype="float32") self.check_operation(a, b, c, '-') - def test_tensor_mul_tensor(self): + def test_scalar_sub_tensor(self): + with _test_eager_guard(): + self.func_scalar_sub_tensor() + self.func_scalar_sub_tensor() + + def func_tensor_mul_tensor(self): # tensor(int64) * scalar(int) a = paddle.ones([2, 2, 2], dtype='int64') b = 1 @@ -174,7 +190,12 @@ def test_tensor_mul_tensor(self): c = paddle.full([2, 2, 2], 1.5, dtype="float32") self.check_operation(a, b, c, '*') - def test_tensor_div_scalar(self): + def test_tensor_mul_tensor(self): + with _test_eager_guard(): + self.func_tensor_mul_tensor() + self.func_tensor_mul_tensor() + + def func_tensor_div_scalar(self): # tensor(int64) / scalar(int) a = paddle.ones([2, 2, 2], dtype='int64') b = 2 @@ -205,7 +226,12 @@ def test_tensor_div_scalar(self): c = paddle.full([2, 2, 2], 2, dtype="float32") self.check_operation(a, b, c, '/') - def test_scalar_div_tensor(self): + def test_tensor_div_scalar(self): + with _test_eager_guard(): + self.func_tensor_div_scalar() + self.func_tensor_div_scalar() + + def func_scalar_div_tensor(self): # scalar(int) / tensor(int64) a = 1 b = paddle.full([2, 2, 2], 2, dtype='int64') @@ -230,7 +256,12 @@ def test_scalar_div_tensor(self): c = paddle.full([2, 2, 2], 2, dtype="float32") self.check_operation(a, b, c, '/') - def test_tensor_pow_scalar(self): + def test_scalar_div_tensor(self): + with _test_eager_guard(): + self.func_scalar_div_tensor() + self.func_scalar_div_tensor() + + def func_tensor_pow_scalar(self): # tensor(int64) ** scalar(int) a = paddle.full([2, 2, 2], 2, dtype='int64') b = 3 @@ -255,7 +286,12 @@ def test_tensor_pow_scalar(self): c = paddle.full([2, 2, 2], 8, dtype="float32") self.check_operation(a, b, c, '**') - def test_scalar_pow_tensor(self): + def test_tensor_pow_scalar(self): + with _test_eager_guard(): + self.func_tensor_pow_scalar() + self.func_tensor_pow_scalar() + + def func_scalar_pow_tensor(self): # scalar(int) ** tensor(int64) a = 3 b = paddle.full([2, 2, 2], 2, dtype='int64') @@ -280,15 +316,25 @@ def test_scalar_pow_tensor(self): c = paddle.full([2, 2, 2], 9, dtype="float32") self.check_operation(a, b, c, '**') + def test_scalar_pow_tensor(self): + with _test_eager_guard(): + self.func_scalar_pow_tensor() + self.func_scalar_pow_tensor() + ## TODO: floordiv op kernel doesn't support float - def test_tensor_floordiv_scalar(self): + def func_tensor_floordiv_scalar(self): # tensor(int64) // scalar(int) a = paddle.full([2, 2, 2], 3, dtype='int64') b = 2 c = paddle.full([2, 2, 2], 1, dtype="int64") self.check_operation(a, b, c, '//') - def test_tensor_mod_scalar(self): + def test_tensor_floordiv_scalar(self): + with _test_eager_guard(): + self.func_tensor_floordiv_scalar() + self.func_tensor_floordiv_scalar() + + def func_tensor_mod_scalar(self): # tensor(int64) % scalar(int) a = paddle.full([2, 2, 2], 3, dtype='int64') b = 2 @@ -313,6 +359,11 @@ def test_tensor_mod_scalar(self): c = paddle.full([2, 2, 2], 1, dtype="float32") self.check_operation(a, b, c, '%') + def test_tensor_mod_scalar(self): + with _test_eager_guard(): + self.func_tensor_mod_scalar() + self.func_tensor_mod_scalar() + if __name__ == '__main__': unittest.main() diff --git a/python/paddle/fluid/tests/unittests/white_list/op_threshold_white_list.py b/python/paddle/fluid/tests/unittests/white_list/op_threshold_white_list.py index 5deca1dc5acd4..91731c1dd0b21 100644 --- a/python/paddle/fluid/tests/unittests/white_list/op_threshold_white_list.py +++ b/python/paddle/fluid/tests/unittests/white_list/op_threshold_white_list.py @@ -51,6 +51,7 @@ 'matrix_power', \ 'cholesky_solve', \ 'solve', \ + 'qr', \ ] NEED_FIX_FP64_CHECK_OUTPUT_THRESHOLD_OP_LIST = ['bilinear_interp',\ diff --git a/python/paddle/fluid/transpiler/collective.py b/python/paddle/fluid/transpiler/collective.py index ea88a89e68224..95ab446e1de6d 100644 --- a/python/paddle/fluid/transpiler/collective.py +++ b/python/paddle/fluid/transpiler/collective.py @@ -42,6 +42,7 @@ def __init__(self, nrings): self.nrings = nrings self.endpoints = None self.current_endpoint = None + self.other_endpoints = None self.nranks = None self.rank = None self.startup_program = None @@ -79,6 +80,12 @@ def transpile(self, startup_program, main_program, rank, endpoints, self.endpoints = endpoints self.current_endpoint = current_endpoint + if current_endpoint: + nranks = len(endpoints) + other_endpoints = endpoints[:] + other_endpoints.remove(current_endpoint) + self.other_endpoints = other_endpoints + self.wait_port = wait_port self.startup_program._origin_program = self.startup_program.clone() @@ -462,9 +469,41 @@ def _transpile_startup_program(self): self.rank, ring_id, self.wait_port, True) else: - print("begin to _transpile_startup_program for single-node") - block = self.startup_program.global_block() - block.append_op(type='c_comm_init_all', attrs={'ring_id': 0}) + if "xpu" in self.trans_mode: + print( + "begin to _transpile_startup_program for single-node in XPU") + block = self.startup_program.global_block() + comm_id_var = block.create_var( + name=unique_name.generate('comm_id'), + persistable=True, + type=core.VarDesc.VarType.RAW) + block.append_op( + type='c_gen_bkcl_id', + inputs={}, + outputs={'Out': comm_id_var}, + attrs={ + 'rank': self.rank, + 'endpoint': self.current_endpoint, + 'other_endpoints': self.other_endpoints, + 'ring_id': 0, + self.op_role_key: OpRole.Forward + }) + block.append_op( + type='c_comm_init', + inputs={'X': comm_id_var}, + outputs={}, + attrs={ + 'nranks': + len(os.getenv("FLAGS_selected_gpus").split(",")), + 'rank': self.rank, + 'ring_id': 0, + self.op_role_key: OpRole.Forward + }) + + else: + print("begin to _transpile_startup_program for single-node") + block = self.startup_program.global_block() + block.append_op(type='c_comm_init_all', attrs={'ring_id': 0}) def _transpile_main_program(self): self._insert_scale_loss_grad_ops() diff --git a/python/paddle/framework/io.py b/python/paddle/framework/io.py index 4f836d94b34eb..c1891d24b88c9 100644 --- a/python/paddle/framework/io.py +++ b/python/paddle/framework/io.py @@ -34,6 +34,10 @@ from paddle.fluid.dygraph.jit import _SaveLoadConfig from paddle.fluid.dygraph.io import _construct_program_holders, _construct_params_and_buffers from paddle.fluid.dygraph.io import INFER_MODEL_SUFFIX, INFER_PARAMS_SUFFIX, INFER_PARAMS_INFO_SUFFIX +try: + from collections.abc import Iterable +except: + from collections import Iterable __all__ = [] @@ -424,7 +428,7 @@ def _parse_every_object(obj, condition_func, convert_func): elif type(obj) == set: return set(_parse_every_object(list(obj), condition_func, convert_func)) else: - if isinstance(obj, collections.Iterable) and not isinstance( + if isinstance(obj, Iterable) and not isinstance( obj, (str, np.ndarray, core.VarBase, core.eager.Tensor, core.LoDTensor)): raise NotImplementedError( diff --git a/python/paddle/incubate/__init__.py b/python/paddle/incubate/__init__.py index d8cc322a66e27..ff7a167f1a670 100644 --- a/python/paddle/incubate/__init__.py +++ b/python/paddle/incubate/__init__.py @@ -29,6 +29,7 @@ from .tensor import segment_min from .passes import fuse_resnet_unit_pass import paddle.incubate.autograd +import paddle.incubate.autotune from . import nn #noqa: F401 diff --git a/python/paddle/incubate/autotune.py b/python/paddle/incubate/autotune.py new file mode 100644 index 0000000000000..e98a23bc52d65 --- /dev/null +++ b/python/paddle/incubate/autotune.py @@ -0,0 +1,157 @@ +# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import paddle +import json +import warnings +from paddle.fluid import core + +__all__ = ['set_config'] + + +def set_config(config=None): + r""" + Set the configuration for kernel, layout and dataloader auto-tuning. + + 1. kernel: When it is enabled, exhaustive search method will be used to select + and cache the best algorithm for the operator in the tuning iteration. Tuning + parameters are as follows: + + - enable(bool): Whether to enable kernel tuning. + - tuning_range(list): Start and end iteration for auto-tuning. Default: [1, 10]. + + 2. layout: When it is enabled, the best data layout such as NCHW or NHWC will be + determined based on the device and data type. When the origin layout setting is + not best, layout transformation will be automaticly performed to improve model + performance. Layout auto-tuning only supports dygraph mode currently. Tuning + parameters are as follows: + + - enable(bool): Whether to enable layout tuning. + + 3. dataloader: When it is enabled, the best num_workers will be selected to replace + the origin dataloader setting. Tuning parameters are as follows: + + - enable(bool): Whether to enable dataloader tuning. + + Args: + config (dict|str|None, optional): Configuration for auto-tuning. If it is a + dictionary, the key is the tuning type, and the value is a dictionary + of the corresponding tuning parameters. If it is a string, the path of + a json file will be specified and the tuning configuration will be set + by the the json file. Default: None, auto-tuning for kernel, layout and + dataloader will be enabled. + + Examples: + .. code-block:: python + :name: auto-tuning + + import paddle + import json + + # config is a dict. + config = { + "kernel": { + "enable": True, + "tuning_range": [1, 5], + }, + "layout": { + "enable": True, + }, + "dataloader": { + "enable": True, + } + } + paddle.incubate.autotune.set_config(config) + + # config is the path of json file. + config_json = json.dumps(config) + with open('config.json', 'w') as json_file: + json_file.write(config_json) + paddle.incubate.autotune.set_config('config.json') + + """ + if config is None: + core.enable_autotune() + core.enable_layout_autotune() + paddle.fluid.reader.set_autotune_config(use_autotune=True) + return + + config_dict = {} + if isinstance(config, dict): + config_dict = config + elif isinstance(config, str): + try: + with open(config, 'r') as filehandle: + config_dict = json.load(filehandle) + except Exception as e: + print('Load config error: {}'.format(e)) + warnings.warn("Use default configuration for auto-tuning.") + + if "kernel" in config_dict: + kernel_config = config_dict["kernel"] + if "enable" in kernel_config: + if isinstance(kernel_config['enable'], bool): + if kernel_config['enable']: + core.enable_autotune() + else: + core.disable_autotune() + else: + warnings.warn( + "The auto-tuning configuration of the kernel is incorrect." + "The `enable` should be bool. Use default parameter instead." + ) + if "tuning_range" in kernel_config: + if isinstance(kernel_config['tuning_range'], list): + tuning_range = kernel_config['tuning_range'] + assert len(tuning_range) == 2 + core.set_autotune_range(tuning_range[0], tuning_range[1]) + else: + warnings.warn( + "The auto-tuning configuration of the kernel is incorrect." + "The `tuning_range` should be list. Use default parameter instead." + ) + if "layout" in config_dict: + layout_config = config_dict["layout"] + if "enable" in layout_config: + if isinstance(layout_config['enable'], bool): + if layout_config['enable']: + core.enable_layout_autotune() + else: + core.disable_layout_autotune() + else: + warnings.warn( + "The auto-tuning configuration of the layout is incorrect." + "The `enable` should be bool. Use default parameter instead." + ) + if "dataloader" in config_dict: + dataloader_config = config_dict["dataloader"] + use_autoune = False + if "enable" in dataloader_config: + if isinstance(dataloader_config['enable'], bool): + use_autoune = dataloader_config['enable'] + else: + warnings.warn( + "The auto-tuning configuration of the dataloader is incorrect." + "The `enable` should be bool. Use default parameter instead." + ) + if "tuning_steps" in dataloader_config: + if isinstance(dataloader_config['tuning_steps'], int): + paddle.fluid.reader.set_autotune_config( + use_autoune, dataloader_config['tuning_steps']) + else: + warnings.warn( + "The auto-tuning configuration of the dataloader is incorrect." + "The `tuning_steps` should be int. Use default parameter instead." + ) + paddle.fluid.reader.set_autotune_config(use_autoune) diff --git a/python/paddle/incubate/distributed/models/moe/__init__.py b/python/paddle/incubate/distributed/models/moe/__init__.py index e1663029ef1f8..fd06b4b8e5287 100644 --- a/python/paddle/incubate/distributed/models/moe/__init__.py +++ b/python/paddle/incubate/distributed/models/moe/__init__.py @@ -11,3 +11,8 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. + +from .gate import GShardGate, BaseGate, SwitchGate, NaiveGate +from .moe_layer import MoELayer +from .grad_clip import ClipGradForMOEByGlobalNorm +ClipGradByGlobalNorm = ClipGradForMOEByGlobalNorm diff --git a/python/paddle/incubate/distributed/models/moe/moe_layer.py b/python/paddle/incubate/distributed/models/moe/moe_layer.py index eebb635e3ead7..ba22ffee3e4d6 100644 --- a/python/paddle/incubate/distributed/models/moe/moe_layer.py +++ b/python/paddle/incubate/distributed/models/moe/moe_layer.py @@ -399,7 +399,7 @@ def forward(self, inp): def experts_fwd(x, fwd_expert_count, experts): if x.shape[0] == 0: - return paddle.empty(x.shape, x.dtype) + return x y = [] last_index = 0 assert isinstance(fwd_expert_count, np.ndarray) @@ -411,7 +411,7 @@ def experts_fwd(x, fwd_expert_count, experts): last_index = expert_count + last_index return paddle.concat(y, axis=0) - if self.recompute_interval <= 0: + if self.recompute_interval <= 0 or x.shape[0] == 0: x = experts_fwd(x, fwd_expert_count.numpy(), self.experts) else: x = _hp_recompute(experts_fwd, x, diff --git a/python/paddle/incubate/nn/__init__.py b/python/paddle/incubate/nn/__init__.py index f359ec1e0d842..43fcabf97317e 100644 --- a/python/paddle/incubate/nn/__init__.py +++ b/python/paddle/incubate/nn/__init__.py @@ -15,10 +15,11 @@ from .layer.fused_transformer import FusedMultiHeadAttention # noqa: F401 from .layer.fused_transformer import FusedFeedForward # noqa: F401 from .layer.fused_transformer import FusedTransformerEncoderLayer # noqa: F401 +from .layer.fused_transformer import FusedMultiTransformer # noqa: F401 __all__ = [ #noqa 'FusedMultiHeadAttention', 'FusedFeedForward', 'FusedTransformerEncoderLayer', - + 'FusedMultiTransformer', ] diff --git a/python/paddle/incubate/nn/functional/__init__.py b/python/paddle/incubate/nn/functional/__init__.py index 4d1c3eee025b0..4da090487785b 100644 --- a/python/paddle/incubate/nn/functional/__init__.py +++ b/python/paddle/incubate/nn/functional/__init__.py @@ -14,5 +14,10 @@ from .fused_transformer import fused_multi_head_attention from .fused_transformer import fused_feedforward +from .fused_transformer import fused_multi_transformer -__all__ = ['fused_multi_head_attention', 'fused_feedforward'] +__all__ = [ + 'fused_multi_head_attention', + 'fused_feedforward', + 'fused_multi_transformer', +] diff --git a/python/paddle/incubate/nn/functional/fused_transformer.py b/python/paddle/incubate/nn/functional/fused_transformer.py index 800d5e832f1ae..3e263f1c6d3ae 100644 --- a/python/paddle/incubate/nn/functional/fused_transformer.py +++ b/python/paddle/incubate/nn/functional/fused_transformer.py @@ -488,3 +488,238 @@ def fused_multi_head_attention(x, attrs=attrs) return (final_out, cache_kv_out) if cache_kv else final_out + + +def fused_multi_transformer(x, + ln_scales, + ln_biases, + qkv_weights, + qkv_biases, + linear_weights, + linear_biases, + ffn_ln_scales, + ffn_ln_biases, + ffn1_weights, + ffn1_biases, + ffn2_weights, + ffn2_biases, + pre_layer_norm=True, + epsilon=1e-05, + cache_kvs=None, + time_step=None, + attn_mask=None, + dropout_rate=0.0, + activation="gelu", + training=False, + mode='upscale_in_train', + ring_id=-1, + name=None): + r""" + This is a fusion operator to compute multi transformer layers in transformer model architecture. + This operator only supports running on GPU. The function of the transformer layer is consistent + with the following pseudo code: + + .. code-block:: python + + if pre_layer_norm: + out = layer_norm(x) + out = qkv_linear(out) + qkv_bias + else: + out = qkv_linear(x) + qkv_bias + out = transpose(out, perm=[2, 0, 3, 1, 4]) + # extract q, k and v from out. + q = out[0:1, ::] + k = out[1:2, ::] + v = out[2:3, ::] + out = q * k^t + out = attn_mask + out + out = softmax(out) + out = dropout(out) + out = out * v + out = transpose(out, perm=[0, 2, 1, 3]) + out = linear(out) + if pre_layer_norm: + out = x + dropout(out + bias) + else: + out = layer_norm(x + dropout(out + bias)) + + residual = out; + if pre_layer_norm: + out = ffn_layer_norm(out) + out = ffn1_linear(out) + out = dropout(activation(out + ffn1_bias)) + out = ffn2_linear(out) + out = residual + dropout(out + ffn2_bias) + if not pre_layer_norm: + out = ffn_layer_norm(out) + + Args: + x (Tensor): the input tensor could be 3-D tensor, the input data type could be float16 or float32, the shape is `[batch\_size, sequence\_length, d\_model]`. + ln_scales (list(Tensor)|tuple(Tensor)): The weight tensors of attention layer_norm, the shape is `[d\_model]`. + ln_biases (list(Tensor)|tuple(Tensor)): The bias tensors of attention layer_norm. the shape is `[d\_model]`. + qkv_weights (list(Tensor)|tuple(Tensor)): The weight tensors of attention qkv computation. The shape is `[3, num\_head, dim\_head, d\_model]`. + qkv_biases (list(Tensor)|tuple(Tensor)|None): The bias tensors of attention qkv computation. The shape is `[3, num\_head, dim\_head]`. + linear_weights (list(Tensor)|tuple(Tensor)): The weight tensors of attention linear. The shape is `[num\_head * dim\_head, d\_model]`. + linear_biases (list(Tensor)|tuple(Tensor)|None): The bias tensors of attention linear. The shape is `[d\_model]`. + ffn_ln_scales (list(Tensor)|tuple(Tensor)): The weight tensors of feedforward layer_norm, the shape is `[d\_model]` + ffn_ln_biases (list(Tensor)|tuple(Tensor)): The bias tensors of feedforward layer_norm, the shape is `[d\_model]` + ffn1_weights (list(Tensor)|tuple(Tensor)): The weight tensors of feedforward first linear, the shape is `[d\_model, dim\_feedforward]`. + ffn1_biases (list(Tensor)|tuple(Tensor)|None): The bias tensors of feedforward first linear, the shape is `[dim\_feedforward]`. + ffn2_weights (list(Tensor)|tuple(Tensor)): The weight tensors of feedforward second linear, the shape is `[dim\_feedforward, d\_model]`. + ffn2_biases (list(Tensor)|tuple(Tensor)|None): The bias tensors of feedforward second linear, the shape is `[d_model]`. + pre_layer_norm (bool, optional): whether it is pre_layer_norm(True) or post_layer_norm(False). Default True. + epsilon (float, optional): Small float value added to denominator of the layer_norm to avoid dividing by zero. Default is 1e-5. + cache_kvs (list(Tensor)|tuple(Tensor), optional): The cache structure tensors for the generation model. The shape is `[2, bsz, num\_head, max\_seq\_len, head\_dim]`. Default None. + time_step (Tensor, optional): The time step tensor for the generation model. Which used in decode stage, to represent the time step, that is, the real seq_len of CacheKV. The shape is `[1]`, must be in CPUPlace. Default None. + attn_mask (Tensor, optional): A tensor used in multi-head attention to prevents attention to + some unwanted positions, usually the paddings or the subsequent positions. It is a tensor + with shape `[batch_size, 1, sequence_length, sequence_length]`. Default None. + dropout_rate (float, optional): The dropout probability of setting units to zero. Default 0.0. + activation (str, optional): The activation. Default "gelu". + training (bool, optional): A flag indicating whether it is in train phrase or not. Default False. + mode (str, optional): ['upscale_in_train'(default) | 'downscale_in_infer'] + + 1. upscale_in_train(default), upscale the output at training time + + - train: out = input * mask / ( 1.0 - p ) + - inference: out = input + + 2. downscale_in_infer, downscale the output at inference + + - train: out = input * mask + - inference: out = input * (1.0 - p) + ring_id (int, optional): For distributed forward in tensor model parallel, only support NCCL. Default is -1, means not using mp. + name (str, optional): Name for the operation (optional, default is None). For more information, please refer to :ref:`api_guide_Name`. + + Returns: + Tensor|tuple: If `cache_kvs` is None, return a tensor that has + the same shape and data type with `x`, representing the output + of Transformer layers. If `cache_kvs` is not None, return the + tuple (output, cache_kvs), which output is the output of + Transformer layers, cache_kvs is inplace with input `cache_kvs`. + + Examples: + .. code-block:: python + + # required: gpu + import paddle + import paddle.incubate.nn.functional as F + import numpy as np + + # input: [batch_size, seq_len, embed_dim] + x = paddle.rand(shape=(2, 4, 128), dtype="float32") + + # ln_scale: [embed_dim], ln_bias: [embed_dim] + ln_scale = paddle.rand(shape=(128,), dtype="float32") + ln_bias = paddle.rand(shape=(128,), dtype="float32") + + # qkv_weight: [3, num_head, head_dim, embed_dim], qkv_bias: [3, num_head, head_dim] + qkv_weight = paddle.rand(shape=(3, 4, 32, 128), dtype="float32") + qkv_bias = paddle.rand(shape=(3, 4, 32), dtype="float32") + + # linear_weight: [embed_dim, embed_dim], linear_bias: [embed_dim] + linear_weight = paddle.rand(shape=(128, 128), dtype="float32") + linear_bias = paddle.rand(shape=(128,), dtype="float32") + + # ffn_ln_scale: [embed_dim], ffn_ln_bias: [embed_dim] + ffn_ln_scale = paddle.rand(shape=(128,), dtype="float32") + ffn_ln_bias = paddle.rand(shape=(128,), dtype="float32") + + # ffn1_weight: [embed_dim, 4*embed_dim], ffn1_bias: [4*embed_dim] + ffn1_weight = paddle.rand(shape=(128, 4*128), dtype="float32") + ffn1_bias = paddle.rand(shape=(4*128,), dtype="float32") + + # ffn2_weight: [4*embed_dim, embed_dim], ffn2_bias: [embed_dim] + ffn2_weight = paddle.rand(shape=(4*128, 128), dtype="float32") + ffn2_bias = paddle.rand(shape=(128,), dtype="float32") + + # self attention mask: [batch_size, 1, seq_len, seq_len] + attn_mask = paddle.rand(shape=(2, 1, 4, 4), dtype="float32") + + # output: [batch_size, seq_len, embed_dim] + output = F.fused_multi_transformer( + x, [ln_scale], [ln_bias], [qkv_weight], [qkv_bias], + [linear_weight], [linear_bias], [ffn_ln_scale], [ffn_ln_bias], + [ffn1_weight], [ffn1_bias], [ffn2_weight], [ffn2_bias], + attn_mask=attn_mask) + # [2, 4, 128] + print(output.shape) + """ + if mode not in ('downscale_in_infer', 'upscale_in_train'): + raise ValueError( + "mode argument should be 'downscale_in_infer' or 'upscale_in_train'") + mode = 'downgrade_in_infer' if mode == 'downscale_in_infer' else mode #semantic transfer + + if _non_static_mode(): + cache_kv_out, final_out = _C_ops.fused_multi_transformer( + x, ln_scales, ln_biases, qkv_weights, qkv_biases, cache_kvs, + time_step, attn_mask, linear_weights, linear_biases, ffn_ln_scales, + ffn_ln_biases, ffn1_weights, ffn1_biases, ffn2_weights, ffn2_biases, + cache_kvs, 'pre_layer_norm', pre_layer_norm, 'epsilon', epsilon, + 'dropout_rate', dropout_rate, 'dropout_is_test', not training, + 'dropout_implementation', mode, 'act_method', activation, 'ring_id', + ring_id) + if cache_kvs is not None: + return final_out, cache_kv_out + return final_out + else: + helper = LayerHelper('fused_multi_transformer', **locals()) + dtype = x.dtype + # check dtypes + check_variable_and_dtype(x, 'x', ['float16', 'float32'], + 'fused_multi_transformer') + check_dtype(dtype, 'dtype', ['float16', 'float32'], + 'fused_multi_transformer') + + # set inputs + inputs = dict() + inputs['X'] = [x] + inputs['LnScale'] = ln_scales + inputs['LnBias'] = ln_biases + inputs['QKVW'] = qkv_weights + if qkv_biases is not None: + inputs['QKVBias'] = qkv_biases + if cache_kvs is not None: + assert len(cache_kvs) == len(qkv_weights) + inputs['CacheKV'] = cache_kvs + if time_step is not None: + inputs['TimeStep'] = time_step + inputs['SrcMask'] = attn_mask + inputs['OutLinearW'] = linear_weights + if linear_biases is not None: + inputs['OutLinearBias'] = linear_biases + + inputs['FFNLnScale'] = ffn_ln_scales + inputs['FFNLnBias'] = ffn_ln_biases + inputs['FFN1Weight'] = ffn1_weights + if ffn1_biases is not None: + inputs['FFN1Bias'] = ffn1_biases + inputs['FFN2Weight'] = ffn2_weights + if ffn2_biases is not None: + inputs['FFN2Bias'] = ffn2_biases + + # set attrs + attrs = { + 'pre_layer_norm': pre_layer_norm, + 'epsilon': epsilon, + 'dropout_rate': dropout_rate, + 'dropout_is_test': not training, + 'dropout_implementation': mode, + 'act_method': activation, + 'ring_id': ring_id + } + + outputs = dict() + final_out = helper.create_variable_for_type_inference(dtype=dtype) + outputs['Out'] = final_out + if cache_kvs: + # NOTE: inplace + outputs['CacheKVOut'] = cache_kvs + + helper.append_op( + type='fused_multi_transformer', + inputs=inputs, + outputs=outputs, + attrs=attrs) + + return (final_out, cache_kvs) if cache_kvs else final_out diff --git a/python/paddle/incubate/nn/layer/fused_transformer.py b/python/paddle/incubate/nn/layer/fused_transformer.py index 28ff9239038bd..a075e1a31a974 100644 --- a/python/paddle/incubate/nn/layer/fused_transformer.py +++ b/python/paddle/incubate/nn/layer/fused_transformer.py @@ -22,6 +22,20 @@ import collections +# for distributed tensor model parallel +def _set_var_distributed(var): + if var is None: + return + + var.is_distributed = True + + # NOTE: use current_block and find_var_recursive to support while_loop + startup_block = paddle.static.default_startup_program().current_block() + main_block = paddle.static.default_main_program().current_block() + startup_block._find_var_recursive(var.name).is_distributed = True + main_block._find_var_recursive(var.name).is_distributed = True + + class FusedMultiHeadAttention(Layer): """ Attention mapps queries and a set of key-value pairs to outputs, and @@ -622,3 +636,390 @@ def __init__(self, def forward(self, src, tgt, src_mask=None, tgt_mask=None, memory_mask=None): raise NotImplementedError() + + +class FusedMultiTransformer(Layer): + """ + FusedMultiTransformer is composed of multi transformer layers which contains two + sub-layers which are self (multi-head) attention and feedforward network. The + function of one transformer layer is consistent with the following pseudo code: + + .. code-block:: python + + if pre_layer_norm: + out = layer_norm(x) + out = qkv_linear(out) + qkv_bias + else: + out = qkv_linear(x) + qkv_bias + out = transpose(out, perm=[2, 0, 3, 1, 4]) + # extract q, k and v from out. + q = out[0:1, ::] + k = out[1:2, ::] + v = out[2:3, ::] + out = q * k^t + out = attn_mask + out + out = softmax(out) + out = dropout(out) + out = out * v + out = transpose(out, perm=[0, 2, 1, 3]) + out = linear(out) + if pre_layer_norm: + out = x + dropout(out + bias) + else: + out = layer_norm(x + dropout(out + bias)) + + residual = out; + if pre_layer_norm: + out = ffn_layer_norm(out) + out = ffn1_linear(out) + out = dropout(activation(out + ffn1_bias)) + out = ffn2_linear(out) + out = residual + dropout(out + ffn2_bias) + if not pre_layer_norm: + out = ffn_layer_norm(out) + + Parameters: + embed_dim (int): The expected feature size in the input and output. + num_heads (int): The number of heads in multi-head attention(MHA). + dim_feedforward (int): The hidden layer size in the feedforward network(FFN). + dropout_rate (float, optional): The dropout probability used in pre-process + and post-precess of MHA and FFN sub-layer. Default 0.0 + activation (str, optional): The activation function in the feedforward + network. Default "gelu". + normalize_before (bool, optional): Indicate whether to put layer normalization + into preprocessing of MHA and FFN sub-layers. If True, pre-process is layer + normalization and post-precess includes dropout, residual connection. + Otherwise, no pre-process and post-precess includes dropout, residual + connection, layer normalization. Default True + ln_scale_attrs(ParamAttr|list|tuple, optional): To specify the weight parameter property + for Attention layer_norm. For Attention layer_norm weight, if it is a list/tuple, `attrs[0]` + would be used as `attr` for transformer layer 0, and `attrs[1]` would be used as + `attr` for transformer layer 1，etc. Otherwise, all layers both use it as + `attr` to create parameters. Default: None, which means the default weight + parameter property is used. See usage for details in :code:`ParamAttr`. + ln_bias_attrs(ParamAttr|list|tuple|bool, optional): To specify the bias parameter property + for Attention layer_norm. For Attention layer_norm bias, if it is a list/tuple, `attrs[0]` + would be used as `attr` for transformer layer 0, and `attrs[1]` would be used as + `attr` for transformer layer 1，etc. Otherwise, all layers both use it as + `attr` to create parameters. The `False` value means the corresponding layer would + not have trainable bias parameter. Default: None, which means the default bias + parameter property is used. See usage for details in :code:`ParamAttr`. + qkv_weight_attrs(ParamAttr|list|tuple, optional): To specify the weight parameter property + for Attention qkv computation. For Attention qkv weight, if it is a list/tuple, `attrs[0]` + would be used as `attr` for transformer layer 0, and `attrs[1]` would be used as + `attr` for transformer layer 1，etc. Otherwise, all layers both use it as + `attr` to create parameters. Default: None, which means the default weight + parameter property is used. See usage for details in :code:`ParamAttr`. + qkv_bias_attrs(ParamAttr|list|tuple|bool, optional): To specify the bias parameter property + for Attention qkv computation. For Attention qkv bias, if it is a list/tuple, `attrs[0]` + would be used as `attr` for transformer layer 0, and `attrs[1]` would be used as + `attr` for transformer layer 1，etc. Otherwise, all layers both use it as + `attr` to create parameters. The `False` value means the corresponding layer would + not have trainable bias parameter. Default: None, which means the default bias + parameter property is used. See usage for details in :code:`ParamAttr`. + linear_weight_attrs(ParamAttr|list|tuple, optional): To specify the weight parameter property + for Attention linear. For Attention linear weight, if it is a list/tuple, `attrs[0]` + would be used as `attr` for transformer layer 0, and `attrs[1]` would be used as + `attr` for transformer layer 1，etc. Otherwise, all layers both use it as + `attr` to create parameters. Default: None, which means the default weight + parameter property is used. See usage for details in :code:`ParamAttr`. + linear_bias_attrs(ParamAttr|list|tuple|bool, optional): To specify the bias parameter property + for Attention linear computation. For Attention linear bias, if it is a list/tuple, `attrs[0]` + would be used as `attr` for transformer layer 0, and `attrs[1]` would be used as + `attr` for transformer layer 1，etc. Otherwise, all layers both use it as + `attr` to create parameters. The `False` value means the corresponding layer would + not have trainable bias parameter. Default: None, which means the default bias + parameter property is used. See usage for details in :code:`ParamAttr`. + ffn_ln_scale_attrs(ParamAttr|list|tuple, optional): To specify the weight parameter property + for FFN layer_norm. For FFN layer_norm weight, if it is a list/tuple, `attrs[0]` + would be used as `attr` for transformer layer 0, and `attrs[1]` would be used as + `attr` for transformer layer 1，etc. Otherwise, all layers both use it as + `attr` to create parameters. Default: None, which means the default weight + parameter property is used. See usage for details in :code:`ParamAttr`. + ffn_ln_bias_attrs(ParamAttr|list|tuple|bool, optional): To specify the bias parameter property + for FFN layer_norm. For FFN layer_norm bias, if it is a list/tuple, `attrs[0]` + would be used as `attr` for transformer layer 0, and `attrs[1]` would be used as + `attr` for transformer layer 1，etc. Otherwise, all layers both use it as + `attr` to create parameters. The `False` value means the corresponding layer would + not have trainable bias parameter. Default: None, which means the default bias + parameter property is used. See usage for details in :code:`ParamAttr`. + ffn1_weight_attrs(ParamAttr|list|tuple, optional): To specify the weight parameter property + for FFN first linear. For FFN first linear weight, if it is a list/tuple, `attrs[0]` + would be used as `attr` for transformer layer 0, and `attrs[1]` would be used as + `attr` for transformer layer 1，etc. Otherwise, all layers both use it as + `attr` to create parameters. Default: None, which means the default weight + parameter property is used. See usage for details in :code:`ParamAttr`. + ffn1_bias_attrs(ParamAttr|list|tuple|bool, optional): To specify the bias parameter property + for FFN first linear. For FFN first linear bias, if it is a list/tuple, `attrs[0]` + would be used as `attr` for transformer layer 0, and `attrs[1]` would be used as + `attr` for transformer layer 1，etc. Otherwise, all layers both use it as + `attr` to create parameters. The `False` value means the corresponding layer would + not have trainable bias parameter. Default: None, which means the default bias + parameter property is used. See usage for details in :code:`ParamAttr`. + ffn2_weight_attrs(ParamAttr|list|tuple, optional): To specify the weight parameter property + for FFN second linear. For FFN second linear weight, if it is a list/tuple, `attrs[0]` + would be used as `attr` for transformer layer 0, and `attrs[1]` would be used as + `attr` for transformer layer 1，etc. Otherwise, all layers both use it as + `attr` to create parameters. Default: None, which means the default weight + parameter property is used. See usage for details in :code:`ParamAttr`. + ffn2_bias_attrs(ParamAttr|list|tuple|bool, optional): To specify the bias parameter property + for FFN second linear. For FFN second linear bias, if it is a list/tuple, `attrs[0]` + would be used as `attr` for transformer layer 0, and `attrs[1]` would be used as + `attr` for transformer layer 1，etc. Otherwise, all layers both use it as + `attr` to create parameters. The `False` value means the corresponding layer would + not have trainable bias parameter. Default: None, which means the default bias + parameter property is used. See usage for details in :code:`ParamAttr`. + epsilon (float, optional): Small float value added to denominator of the layer_norm to + avoid dividing by zero. Default: 1e-05. + num_layers (int, optional): The number of layers of the transformer. If `qkv_weight_attrs` + is a list or tuple, the number of layers is obtained from `qkv_weight_attrs`. num_layers + only takes effect when `qkv_weight_attrs` is not a list or tuple. Default: -1. + nranks (int, optional): Distributed tensor model parallel nranks. Default is 1, means not using mp. + ring_id (int, optional): For distributed tensor model parallel. Default is -1, means not using mp. + name (str, optional): The default value is None. Normally there is no need for user to set + this property. For more information, please refer to :ref:`api_guide_Name`. + + Examples: + + .. code-block:: python + + # required: gpu + import paddle + from paddle.incubate.nn import FusedMultiTransformer + + # encoder input: [batch_size, src_len, d_model] + enc_input = paddle.rand((2, 4, 128)) + # self attention mask: [batch_size, 1, src_len, src_len] + attn_mask = paddle.rand((2, 1, 4, 4)) + encoder_layers = FusedMultiTransformer(128, 2, 512, num_layers=1) + enc_output = encoder_layers(enc_input, attn_mask) # [2, 4, 128] + """ + + def __init__(self, + embed_dim, + num_heads, + dim_feedforward, + dropout_rate=0.0, + activation="gelu", + normalize_before=True, + ln_scale_attrs=None, + ln_bias_attrs=None, + qkv_weight_attrs=None, + qkv_bias_attrs=None, + linear_weight_attrs=None, + linear_bias_attrs=None, + ffn_ln_scale_attrs=None, + ffn_ln_bias_attrs=None, + ffn1_weight_attrs=None, + ffn1_bias_attrs=None, + ffn2_weight_attrs=None, + ffn2_bias_attrs=None, + epsilon=1e-5, + num_layers=-1, + nranks=1, + ring_id=-1, + name=None): + super(FusedMultiTransformer, self).__init__() + + assert embed_dim > 0, ("Expected embed_dim to be greater than 0, " + "but recieved {}".format(embed_dim)) + assert num_heads > 0, ("Expected nhead to be greater than 0, " + "but recieved {}".format(num_heads)) + assert dim_feedforward > 0, ( + "Expected dim_feedforward to be greater than 0, but recieved {}". + format(dim_feedforward)) + + self.normalize_before = normalize_before + self._dtype = self._helper.get_default_dtype() + self._epsilon = epsilon + self._ring_id = ring_id + + self.embed_dim = embed_dim + self.num_heads = num_heads + self.head_dim = embed_dim // num_heads + assert self.head_dim * num_heads == embed_dim, "embed_dim must be divisible by num_heads" + + # tensor model parallel + if nranks > 1: + assert ring_id != -1 + assert num_heads % nranks == 0 + assert dim_feedforward % nranks == 0 + num_heads = num_heads // nranks + dim_feedforward = dim_feedforward // nranks + self._dim_feedforward = dim_feedforward + + if isinstance(qkv_weight_attrs, (list, tuple)): + num_layers = len(qkv_weight_attrs) + assert num_layers > 0 + + self.ln_scales, self.ln_biases = [], [] + self.qkv_weights, self.qkv_biases = [], [] + self.linear_weights, self.linear_biases = [], [] + self.ffn_ln_scales, self.ffn_ln_biases = [], [] + self.ffn1_weights, self.ffn1_biases = [], [] + self.ffn2_weights, self.ffn2_biases = [], [] + + def get_attr(attrs, idx): + if isinstance(attrs, (list, tuple)): + assert len(attrs) == num_layers + return attrs[idx] + return attrs + + for i in range(num_layers): + ln_scale_attr = get_attr(ln_scale_attrs, i) + ln_bias_attr = get_attr(ln_bias_attrs, i) + qkv_weight_attr = get_attr(qkv_weight_attrs, i) + qkv_bias_attr = get_attr(qkv_bias_attrs, i) + linear_weight_attr = get_attr(linear_weight_attrs, i) + linear_bias_attr = get_attr(linear_bias_attrs, i) + + ffn_ln_scale_attr = get_attr(ffn_ln_scale_attrs, i) + ffn_ln_bias_attr = get_attr(ffn_ln_bias_attrs, i) + ffn1_weight_attr = get_attr(ffn1_weight_attrs, i) + ffn1_bias_attr = get_attr(ffn1_bias_attrs, i) + ffn2_weight_attr = get_attr(ffn2_weight_attrs, i) + ffn2_bias_attr = get_attr(ffn2_bias_attrs, i) + + ln_scale = self.create_parameter( + attr=ln_scale_attr, + shape=[embed_dim], + default_initializer=Constant(value=1.0)) + ln_bias = self.create_parameter( + attr=ln_bias_attr, shape=[embed_dim], is_bias=True) + qkv_weight = self.create_parameter( + shape=[3, num_heads, self.head_dim, embed_dim], + attr=qkv_weight_attr, + dtype=self._dtype, + is_bias=False) + qkv_bias = self.create_parameter( + shape=[3, num_heads, self.head_dim], + attr=qkv_bias_attr, + dtype=self._dtype, + is_bias=True) + linear_weight = self.create_parameter( + shape=[num_heads * self.head_dim, embed_dim], + attr=linear_weight_attr, + dtype=self._dtype, + is_bias=False) + linear_bias = self.create_parameter( + shape=[embed_dim], + attr=linear_bias_attr, + dtype=self._dtype, + is_bias=True) + + ffn_ln_scale = self.create_parameter( + shape=[embed_dim], + attr=ffn_ln_scale_attr, + is_bias=False, + default_initializer=Constant(1.0)) + ffn_ln_bias = self.create_parameter( + shape=[embed_dim], attr=ffn_ln_bias_attr, is_bias=True) + ffn1_weight = self.create_parameter( + shape=[embed_dim, dim_feedforward], + attr=ffn1_weight_attr, + dtype=self._dtype, + is_bias=False) + ffn1_bias = self.create_parameter( + shape=[dim_feedforward], + attr=ffn1_bias_attr, + dtype=self._dtype, + is_bias=True) + ffn2_weight = self.create_parameter( + shape=[dim_feedforward, embed_dim], + attr=ffn2_weight_attr, + dtype=self._dtype, + is_bias=False) + ffn2_bias = self.create_parameter( + shape=[embed_dim], + attr=ffn2_bias_attr, + dtype=self._dtype, + is_bias=True) + + # tensor model parallel + if nranks > 1: + # column parallel + _set_var_distributed(qkv_weight) + _set_var_distributed(qkv_bias) + _set_var_distributed(ffn1_weight) + _set_var_distributed(ffn1_bias) + # row parallel + _set_var_distributed(linear_weight) + _set_var_distributed(ffn2_weight) + + self.ln_scales.append(ln_scale) + self.ln_biases.append(ln_bias) + self.qkv_weights.append(qkv_weight) + self.qkv_biases.append(qkv_bias) + self.linear_weights.append(linear_weight) + self.linear_biases.append(linear_bias) + + self.ffn_ln_scales.append(ffn_ln_scale) + self.ffn_ln_biases.append(ffn_ln_bias) + self.ffn1_weights.append(ffn1_weight) + self.ffn1_biases.append(ffn1_bias) + self.ffn2_weights.append(ffn2_weight) + self.ffn2_biases.append(ffn2_bias) + + self.dropout_rate = dropout_rate + self.activation = activation + self.name = name + + def forward(self, src, attn_mask=None, caches=None, time_step=None): + """ + Applies multi transformer layers on the input. + + Parameters: + src (Tensor): The input of Transformer layers. It is + a tensor with shape `[batch_size, sequence_length, d_model]`. + The data type should be float16 or float32. + attn_mask (Tensor, optional): A tensor used in multi-head attention + to prevents attention to some unwanted positions, usually the + paddings or the subsequent positions. It is a tensor with shape + `[batch_size, 1, sequence_length, sequence_length]`. It can be + None when nothing wanted or needed to be prevented attention to. + Default None. + caches (list(Tensor)|tuple(Tensor), optional): The cache structure + tensors for the inference generation model. It is only used for + inference and should be None for training. The shape is + `[2, batch_size, num_head, max_seq_len, head_dim]`. Default None. + time_step (Tensor, optional): The time step tensor for the generation + model. Which used in decode stage, to represent the time step, + that is, the real seq_len of CacheKV. The shape is `[1]`, must be + in CPUPlace. Default None. + + Returns: + Tensor|tuple: If `caches` is None, return a tensor that has + the same shape and data type with `src`, representing the output + of Transformer layers. If `caches` is not None, return the + tuple (output, caches), which output is the output of + Transformer layers, caches is inplace with input `caches`. + """ + + if caches is not None: + assert len(caches) == len(self.qkv_weights) + out = incubate_f.fused_multi_transformer( + src, + self.ln_scales, + self.ln_biases, + self.qkv_weights, + self.qkv_biases, + self.linear_weights, + self.linear_biases, + self.ffn_ln_scales, + self.ffn_ln_biases, + self.ffn1_weights, + self.ffn1_biases, + self.ffn2_weights, + self.ffn2_biases, + pre_layer_norm=self.normalize_before, + epsilon=self._epsilon, + cache_kvs=caches, + time_step=time_step, + attn_mask=attn_mask, + dropout_rate=self.dropout_rate, + activation=self.activation, + training=self.training, + mode='upscale_in_train', + ring_id=self._ring_id, + name=self.name) + return out diff --git a/python/paddle/incubate/operators/resnet_unit.py b/python/paddle/incubate/operators/resnet_unit.py index cba1d4863cbd4..4ddcfbac8791f 100644 --- a/python/paddle/incubate/operators/resnet_unit.py +++ b/python/paddle/incubate/operators/resnet_unit.py @@ -34,7 +34,6 @@ from paddle.fluid.data_feeder import convert_dtype from paddle.fluid.param_attr import ParamAttr from paddle import _C_ops -__all__ = ['resnet_unit', 'ResNetUnit'] def resnet_unit(x, filter_x, scale_x, bias_x, mean_x, var_x, z, filter_z, diff --git a/python/paddle/incubate/optimizer/distributed_fused_lamb.py b/python/paddle/incubate/optimizer/distributed_fused_lamb.py index 74b5398230dee..4d40a477ffc07 100644 --- a/python/paddle/incubate/optimizer/distributed_fused_lamb.py +++ b/python/paddle/incubate/optimizer/distributed_fused_lamb.py @@ -38,6 +38,7 @@ def __init__(self, is_grad_scaled_by_nranks=True, alignment=128, use_master_param_norm=True, + gradient_accumulation_steps=1, name=None): assert not framework._non_static_mode( ), "DistributedFusedLamb does not support dygraph mode" @@ -63,6 +64,9 @@ def __init__(self, self._scale = None self._ring_id = 0 self._use_master_param_norm = use_master_param_norm + self._gradient_accumulation_steps = gradient_accumulation_steps + assert self._gradient_accumulation_steps >= 1 + self.helper = LayerHelper('distributed_fused_lamb') self._supports_check_nan_inf = True # very import flag for AMP @@ -73,8 +77,19 @@ def __init__(self, dtype=core.VarDesc.VarType.BOOL) self._step = None + if self._gradient_accumulation_steps > 1: + self._stop_update = main_block.create_var( + name=unique_name.generate('stop_update'), + shape=[1], + dtype=core.VarDesc.VarType.BOOL) + else: + self._stop_update = None + self._param_to_master_param = {} + def _get_stop_update_var(self): + return self._stop_update if self._stop_update is not None else False + def _set_step(self, step): self._step = step @@ -194,6 +209,20 @@ def _apply_gradients_impl(self, params_grads): param_order = self._create_persistable_var('param_order', dtype='int32') param_order.is_distributed = True + if self._gradient_accumulation_steps > 1: + fp32_acc_fused_grad = [ + self._create_persistable_var('fp32_acc_fused_grad') + ] + fp16_acc_fused_grad = [ + self._create_persistable_var( + 'fp16_acc_fused_grad', dtype='float16') + ] + acc_step = [self._create_persistable_var('acc_step', dtype='int64')] + else: + fp32_acc_fused_grad = [] + fp16_acc_fused_grad = [] + acc_step = [] + step = self._get_or_create_step() rank = get_rank() @@ -298,6 +327,11 @@ def _apply_gradients_impl(self, params_grads): 'ParamOut': params, 'GradOut': grads, 'FoundInf': [self._found_inf], + 'FP32AccFusedGrad': fp32_acc_fused_grad, + 'FP16AccFusedGrad': fp16_acc_fused_grad, + 'AccStep': acc_step, + 'StopUpdate': self._stop_update + if self._stop_update is not None else [], 'Step': [step], }, attrs={ @@ -311,5 +345,6 @@ def _apply_gradients_impl(self, params_grads): 'ring_id': self._ring_id, 'use_master_param_norm': self._use_master_param_norm, 'is_grad_scaled_by_nranks': self._is_grad_scaled_by_nranks, + 'acc_steps': self._gradient_accumulation_steps, }) return [lamb_op] diff --git a/python/paddle/nn/__init__.py b/python/paddle/nn/__init__.py index b4824eff007d6..bceee4b964a33 100644 --- a/python/paddle/nn/__init__.py +++ b/python/paddle/nn/__init__.py @@ -138,6 +138,8 @@ from .layer.distance import PairwiseDistance # noqa: F401 from .layer.vision import PixelShuffle # noqa: F401 +from .layer.vision import PixelUnshuffle # noqa: F401 +from .layer.vision import ChannelShuffle # noqa: F401 from .layer.container import LayerDict # noqa: F401 from .utils.spectral_norm_hook import spectral_norm @@ -300,6 +302,8 @@ def weight_norm(*args): 'Swish', 'Mish', 'PixelShuffle', + 'PixelUnshuffle', + 'ChannelShuffle', 'ELU', 'ReLU6', 'LayerDict', diff --git a/python/paddle/nn/functional/__init__.py b/python/paddle/nn/functional/__init__.py index a24afc45a5995..68213d831c550 100644 --- a/python/paddle/nn/functional/__init__.py +++ b/python/paddle/nn/functional/__init__.py @@ -114,6 +114,8 @@ from .vision import affine_grid # noqa: F401 from .vision import grid_sample # noqa: F401 from .vision import pixel_shuffle # noqa: F401 +from .vision import pixel_unshuffle # noqa: F401 +from .vision import channel_shuffle # noqa: F401 from .input import one_hot # noqa: F401 from .input import embedding # noqa: F401 from ...fluid.layers import gather_tree # noqa: F401 @@ -213,6 +215,8 @@ 'grid_sample', 'local_response_norm', 'pixel_shuffle', + 'pixel_unshuffle', + 'channel_shuffle', 'embedding', 'gather_tree', 'one_hot', diff --git a/python/paddle/nn/functional/pooling.py b/python/paddle/nn/functional/pooling.py index b9cae4784725d..3160f04e830d2 100755 --- a/python/paddle/nn/functional/pooling.py +++ b/python/paddle/nn/functional/pooling.py @@ -1401,9 +1401,9 @@ def adaptive_avg_pool2d(x, output_size, data_format='NCHW', name=None): output_size[1] = in_w if in_dygraph_mode(): - return _C_ops.final_state_pool2d(x, output_size, [1, 1], [0, 0], False, - True, data_format, 'avg', False, True, - "EXPLICIT") + return _C_ops.final_state_pool2d_gpudnn_unused( + x, output_size, [1, 1], [0, 0], False, True, data_format, 'avg', + False, True, "EXPLICIT") if _in_legacy_dygraph(): return _C_ops.pool2d(x, 'pooling_type', 'avg', 'ksize', output_size, diff --git a/python/paddle/nn/functional/vision.py b/python/paddle/nn/functional/vision.py index 43c7757a8777b..9a9c2ee4cf7d1 100644 --- a/python/paddle/nn/functional/vision.py +++ b/python/paddle/nn/functional/vision.py @@ -21,6 +21,7 @@ from paddle import _C_ops from ...device import is_compiled_with_rocm from paddle import in_dynamic_mode +from paddle.framework import _non_static_mode __all__ = [] @@ -344,3 +345,129 @@ def pixel_shuffle(x, upscale_factor, data_format="NCHW", name=None): attrs={"upscale_factor": upscale_factor, "data_format": data_format}) return out + + +def pixel_unshuffle(x, downscale_factor, data_format="NCHW", name=None): + """ + This API implements pixel unshuffle operation. + See more details in :ref:`api_nn_vision_PixelUnshuffle` . + + Parameters: + x (Tensor): 4-D tensor, the data type should be float32 or float64. + downscale_factor (int): Factor to decrease spatial resolution. + data_format (str): The data format of the input and output data. An optional string of NCHW or NHWC. The default is NCHW. When it is NCHW, the data is stored in the order of [batch_size, input_channels, input_height, input_width]. + name (str, optional): Name for the operation (optional, default is None). Normally there is no need for user to set this property. For more information, please refer to :ref:`api_guide_Name`. + + Returns: + Out (Tensor): Reshaped tensor according to the new dimension. + + Examples: + .. code-block:: python + :name: pixel_unshuffle-example + + import paddle + import paddle.nn.functional as F + x = paddle.randn([2, 1, 12, 12]) + out = F.pixel_unshuffle(x, 3) + # out.shape = [2, 9, 4, 4] + """ + if len(x.shape) != 4: + raise ValueError( + "Input x should be 4D tensor, but received x with the shape of {}". + format(x.shape)) + + if not isinstance(downscale_factor, int): + raise TypeError("Downscale factor must be int type") + + if downscale_factor <= 0: + raise ValueError("Downscale factor must be positive") + + if data_format not in ["NCHW", "NHWC"]: + raise ValueError("Attr(data_format) should be 'NCHW' or 'NHWC'." + "But recevie Attr(data_format): {} ".format( + data_format)) + + if _non_static_mode(): + return _C_ops.pixel_unshuffle(x, "downscale_factor", downscale_factor, + "data_format", data_format) + + helper = LayerHelper("pixel_unshuffle", **locals()) + check_variable_and_dtype(x, 'x', ['float32', 'float64'], 'pixel_unshuffle') + out = helper.create_variable_for_type_inference(dtype=x.dtype) + helper.append_op( + type="pixel_unshuffle", + inputs={"X": x}, + outputs={"Out": out}, + attrs={ + "downscale_factor": downscale_factor, + "data_format": data_format + }) + return out + + +def channel_shuffle(x, groups, data_format="NCHW", name=None): + """ + This API implements channel shuffle operation. + See more details in :ref:`api_nn_vision_ChannelShuffle` . + + Parameters: + x (Tensor): 4-D tensor, the data type should be float32 or float64. + groups (int): Number of groups to divide channels in. + data_format (str): The data format of the input and output data. An optional string of NCHW or NHWC. The default is NCHW. When it is NCHW, the data is stored in the order of [batch_size, input_channels, input_height, input_width]. + name (str, optional): Name for the operation (optional, default is None). Normally there is no need for user to set this property. For more information, please refer to :ref:`api_guide_Name`. + + Returns: + Out (Tensor): Rearranged tensor keeping the original tensor shape. + + Examples: + .. code-block:: python + :name: channel_shuffle-example + + import paddle + import paddle.nn.functional as F + x = paddle.arange(0, 0.6, 0.1, 'float32') + x = paddle.reshape(x, [1, 6, 1, 1]) + # [[[[0. ]], + # [[0.10000000]], + # [[0.20000000]], + # [[0.30000001]], + # [[0.40000001]], + # [[0.50000000]]]] + y = F.channel_shuffle(x, 3) + # [[[[0. ]], + # [[0.20000000]], + # [[0.40000001]], + # [[0.10000000]], + # [[0.30000001]], + # [[0.50000000]]]] + """ + if len(x.shape) != 4: + raise ValueError( + "Input x should be 4D tensor, but received x with the shape of {}". + format(x.shape)) + + if not isinstance(groups, int): + raise TypeError("groups must be int type") + + if groups <= 0: + raise ValueError("groups must be positive") + + if data_format not in ["NCHW", "NHWC"]: + raise ValueError("Attr(data_format) should be 'NCHW' or 'NHWC'." + "But recevie Attr(data_format): {} ".format( + data_format)) + + if _non_static_mode(): + return _C_ops.channel_shuffle(x, "groups", groups, "data_format", + data_format) + + helper = LayerHelper("channel_shuffle", **locals()) + check_variable_and_dtype(x, 'x', ['float32', 'float64'], 'channel_shuffle') + out = helper.create_variable_for_type_inference(dtype=x.dtype) + helper.append_op( + type="channel_shuffle", + inputs={"X": x}, + outputs={"Out": out}, + attrs={"groups": groups, + "data_format": data_format}) + return out diff --git a/python/paddle/nn/layer/__init__.py b/python/paddle/nn/layer/__init__.py index 7dd18f1fefd65..31364f0281c8a 100644 --- a/python/paddle/nn/layer/__init__.py +++ b/python/paddle/nn/layer/__init__.py @@ -88,6 +88,8 @@ from .norm import LocalResponseNorm # noqa: F401 from .vision import PixelShuffle # noqa: F401 +from .vision import PixelUnshuffle # noqa: F401 +from .vision import ChannelShuffle # noqa: F401 from .distance import PairwiseDistance # noqa: F401 from .container import LayerDict # noqa: F401 diff --git a/python/paddle/nn/layer/rnn.py b/python/paddle/nn/layer/rnn.py index b5daa290456e3..ae6e37a02751d 100644 --- a/python/paddle/nn/layer/rnn.py +++ b/python/paddle/nn/layer/rnn.py @@ -37,6 +37,10 @@ from paddle.framework import core from paddle.static import default_startup_program from paddle.static import program_guard +try: + from collections.abc import Sequence +except: + from collections import Sequence __all__ = [] @@ -197,7 +201,7 @@ def _is_shape_sequence(seq): # TODO: Add check for the illegal if isinstance(seq, dict): return True - return (isinstance(seq, collections.Sequence) and + return (isinstance(seq, Sequence) and not isinstance(seq, six.string_types)) class Shape(object): diff --git a/python/paddle/nn/layer/vision.py b/python/paddle/nn/layer/vision.py index 0531afb4eeeeb..6d5c112d75703 100644 --- a/python/paddle/nn/layer/vision.py +++ b/python/paddle/nn/layer/vision.py @@ -87,3 +87,139 @@ def extra_repr(self): if self._name is not None: main_str += ', name={}'.format(self._name) return main_str + + +class PixelUnshuffle(Layer): + """ + This operator rearranges elements in a tensor of shape :math:`[N, C, H, W]` + to a tensor of shape :math:`[N, r^2C, H/r, W/r]`, or from shape + :math:`[N, H, W, C]` to :math:`[N, H/r, W/r, r^2C]`, where :math:`r` is the + downscale factor. This operation is the reversion of PixelShuffle operation. + Please refer to the paper: `Real-Time Single Image and Video Super-Resolution + Using an Efficient Sub-Pixel Convolutional Neural Network `_ . + by Shi et. al (2016) for more details. + + Parameters: + downscale_factor (int): Factor to decrease spatial resolution. + data_format (str): The data format of the input and output data. An optional string of NCHW or NHWC. The default is NCHW. When it is NCHW, the data is stored in the order of [batch_size, input_channels, input_height, input_width]. + name (str, optional): Name for the operation (optional, default is None). Normally there is no need for user to set this property. For more information, please refer to :ref:`api_guide_Name`. + + Shape: + - **x**: 4-D tensor with shape of :math:`[N, C, H, W]` or :math:`[N, C, H, W]`. + - **out**: 4-D tensor with shape of :math:`[N, r^2C, H/r, W/r]` or :math:`[N, H/r, W/r, r^2C]`, where :math:`r` is :attr:`downscale_factor`. + + Examples: + .. code-block:: python + :name: PixelUnshuffle-example + + import paddle + import paddle.nn as nn + + x = paddle.randn([2, 1, 12, 12]) + pixel_unshuffle = nn.PixelUnshuffle(3) + out = pixel_unshuffle(x) + # out.shape = [2, 9, 4, 4] + + """ + + def __init__(self, downscale_factor, data_format="NCHW", name=None): + super(PixelUnshuffle, self).__init__() + + if not isinstance(downscale_factor, int): + raise TypeError("Downscale factor must be int type") + + if downscale_factor <= 0: + raise ValueError("Downscale factor must be positive") + + if data_format not in ["NCHW", "NHWC"]: + raise ValueError("Data format should be 'NCHW' or 'NHWC'." + "But recevie data format: {}".format(data_format)) + + self._downscale_factor = downscale_factor + self._data_format = data_format + self._name = name + + def forward(self, x): + return functional.pixel_unshuffle(x, self._downscale_factor, + self._data_format, self._name) + + def extra_repr(self): + main_str = 'downscale_factor={}'.format(self._downscale_factor) + if self._data_format != 'NCHW': + main_str += ', data_format={}'.format(self._data_format) + if self._name is not None: + main_str += ', name={}'.format(self._name) + return main_str + + +class ChannelShuffle(Layer): + """ + This operator divides channels in a tensor of shape [N, C, H, W] or [N, H, W, C] into g groups, + getting a tensor with the shape of [N, g, C/g, H, W] or [N, H, W, g, C/g], and transposes them + as [N, C/g, g, H, W] or [N, H, W, g, C/g], then rearranges them to original tensor shape. This + operation can improve the interaction between channels, using features efficiently. Please + refer to the paper: `ShuffleNet: An Extremely Efficient + Convolutional Neural Network for Mobile Devices `_ . + by Zhang et. al (2017) for more details. + + Parameters: + groups (int): Number of groups to divide channels in. + data_format (str): The data format of the input and output data. An optional string of NCHW or NHWC. The default is NCHW. When it is NCHW, the data is stored in the order of [batch_size, input_channels, input_height, input_width]. + name (str, optional): Name for the operation (optional, default is None). Normally there is no need for user to set this property. For more information, please refer to :ref:`api_guide_Name`. + + Shape: + - **x**: 4-D tensor with shape of [N, C, H, W] or [N, H, W, C]. + - **out**: 4-D tensor with shape and dtype same as x. + + Examples: + .. code-block:: python + :name: ChannelShuffle-example + + import paddle + import paddle.nn as nn + x = paddle.arange(0, 0.6, 0.1, 'float32') + x = paddle.reshape(x, [1, 6, 1, 1]) + # [[[[0. ]], + # [[0.10000000]], + # [[0.20000000]], + # [[0.30000001]], + # [[0.40000001]], + # [[0.50000000]]]] + channel_shuffle = nn.ChannelShuffle(3) + y = channel_shuffle(x) + # [[[[0. ]], + # [[0.20000000]], + # [[0.40000001]], + # [[0.10000000]], + # [[0.30000001]], + # [[0.50000000]]]] + """ + + def __init__(self, groups, data_format="NCHW", name=None): + super(ChannelShuffle, self).__init__() + + if not isinstance(groups, int): + raise TypeError("groups must be int type") + + if groups <= 0: + raise ValueError("groups must be positive") + + if data_format not in ["NCHW", "NHWC"]: + raise ValueError("Data format should be 'NCHW' or 'NHWC'." + "But recevie data format: {}".format(data_format)) + + self._groups = groups + self._data_format = data_format + self._name = name + + def forward(self, x): + return functional.channel_shuffle(x, self._groups, self._data_format, + self._name) + + def extra_repr(self): + main_str = 'groups={}'.format(self._groups) + if self._data_format != 'NCHW': + main_str += ', data_format={}'.format(self._data_format) + if self._name is not None: + main_str += ', name={}'.format(self._name) + return main_str diff --git a/python/paddle/profiler/profiler_statistic.py b/python/paddle/profiler/profiler_statistic.py index 422dbe4ce359f..50aa3a1f11f85 100755 --- a/python/paddle/profiler/profiler_statistic.py +++ b/python/paddle/profiler/profiler_statistic.py @@ -13,6 +13,7 @@ # limitations under the License. import collections from enum import Enum +import re from paddle.fluid.core import TracerEventType @@ -1317,10 +1318,11 @@ def format_ratio(ratio, indent=0): append(header_sep) append(row_format.format(*headers)) append(header_sep) + kernel_name_pattern = re.compile('(.+?)(<.*>)(\(.*\))') for row_values in all_row_values: - indx = row_values[0].find('(') - if indx != -1: - name = row_values[0][:indx] + match = kernel_name_pattern.match(row_values[0]) + if match: + name = match.group(1) + match.group(2) else: name = row_values[0] if len(name) > name_column_width: diff --git a/python/paddle/profiler/utils.py b/python/paddle/profiler/utils.py index 6ae3fe4e60b92..fba1aeabf28bd 100644 --- a/python/paddle/profiler/utils.py +++ b/python/paddle/profiler/utils.py @@ -21,6 +21,7 @@ from paddle.fluid.core import (_RecordEvent, TracerEventType) _is_profiler_used = False +_has_optimizer_wrapped = False _AllowedEventTypeList = [ TracerEventType.Dataloader, TracerEventType.ProfileStep, @@ -154,20 +155,31 @@ def load_profiler_result(filename: str): return core.load_profiler_result(filename) +def in_profiler_mode(): + return _is_profiler_used == True + + def wrap_optimizers(): def optimizer_warpper(func): @functools.wraps(func) def warpper(*args, **kwargs): - with RecordEvent( - 'Optimization Step', - event_type=TracerEventType.Optimization): + if in_profiler_mode(): + with RecordEvent( + 'Optimization Step', + event_type=TracerEventType.Optimization): + return func(*args, **kwargs) + else: return func(*args, **kwargs) return warpper + global _has_optimizer_wrapped + if _has_optimizer_wrapped == True: + return import paddle.optimizer as optimizer for classname in optimizer.__all__: if classname != 'Optimizer': classobject = getattr(optimizer, classname) if getattr(classobject, 'step', None) != None: classobject.step = optimizer_warpper(classobject.step) + _has_optimizer_wrapped = True diff --git a/python/paddle/sparse/__init__.py b/python/paddle/sparse/__init__.py index 5e716d69379ed..93653e09c9019 100644 --- a/python/paddle/sparse/__init__.py +++ b/python/paddle/sparse/__init__.py @@ -15,9 +15,14 @@ from .creation import sparse_coo_tensor from .creation import sparse_csr_tensor from .layer.activation import ReLU +from .layer.norm import BatchNorm + from .layer.conv import Conv3D from .layer.conv import SubmConv3D +from .layer.pooling import MaxPool3D + __all__ = [ - 'sparse_coo_tensor', 'sparse_csr_tensor', 'ReLU', 'Conv3D', 'SubmConv3D' + 'sparse_coo_tensor', 'sparse_csr_tensor', 'ReLU', 'Conv3D', 'SubmConv3D', + 'BatchNorm', 'MaxPool3D' ] diff --git a/python/paddle/sparse/creation.py b/python/paddle/sparse/creation.py index d494336e1ff50..2cfbb3144acc2 100644 --- a/python/paddle/sparse/creation.py +++ b/python/paddle/sparse/creation.py @@ -20,6 +20,8 @@ from ..tensor import max from ..fluid.data_feeder import check_variable_and_dtype, check_type, check_dtype, convert_dtype +import numpy as np + __all__ = [ 'sparse_coo_tensor', 'sparse_csr_tensor', @@ -33,11 +35,14 @@ def _handle_dtype(data, dtype): return data -def _infer_dense_shape(indices): +def _infer_dense_shape(indices, values): assert len(indices.shape) == 2 lens = max(indices, axis=1) lens = lens + 1 - return list(lens.numpy()) + lens = lens.numpy() + if len(values.shape) > 1: + lens = np.append(lens, values.shape[1:]) + return list(lens) def _get_place(place): @@ -106,7 +111,7 @@ def sparse_coo_tensor(indices, with _test_eager_guard(): indices = [[0, 1, 2], [1, 2, 0]] values = [1.0, 2.0, 3.0] - dense_shape = [2, 3] + dense_shape = [3, 3] coo = paddle.sparse.sparse_coo_tensor(indices, values, dense_shape) # print(coo) # Tensor(shape=[2, 3], dtype=paddle.float32, place=Place(gpu:0), stop_gradient=True, @@ -145,7 +150,8 @@ def sparse_coo_tensor(indices, values = _handle_dtype(values, dtype) values.stop_gradient = stop_gradient - min_shape = _infer_dense_shape(indices) + min_shape = _infer_dense_shape(indices, values) + if shape is None: shape = min_shape else: diff --git a/python/paddle/sparse/functional/__init__.py b/python/paddle/sparse/functional/__init__.py index 93c3ccda4a613..f1ca4cc6fcc48 100644 --- a/python/paddle/sparse/functional/__init__.py +++ b/python/paddle/sparse/functional/__init__.py @@ -15,5 +15,6 @@ from .activation import relu # noqa: F401 from .conv import conv3d # noqa: F401 from .conv import subm_conv3d # noqa: F401 +from .pooling import max_pool3d # noqa: F401 -__all__ = ['relu', 'conv3d', 'subm_conv3d'] +__all__ = ['relu', 'conv3d', 'subm_conv3d', 'max_pool3d'] diff --git a/python/paddle/sparse/functional/conv.py b/python/paddle/sparse/functional/conv.py index d8c0e5c914ccb..42b7b49835cf0 100644 --- a/python/paddle/sparse/functional/conv.py +++ b/python/paddle/sparse/functional/conv.py @@ -16,6 +16,8 @@ from paddle import _C_ops, in_dynamic_mode from ...fluid.layers.utils import convert_to_list +from ...fluid.layers.nn import elementwise_add +from .. import sparse_coo_tensor from paddle.nn.functional.conv import _update_padding_nd @@ -30,7 +32,6 @@ def _conv3d(x, data_format="NDHWC", name=None): assert in_dynamic_mode(), "Currently, only support dynamic mode" - assert bias == None, "Currently, sparse_conv3d does not support bias" assert groups == 1, "Currently, only support groups=1" dims = 3 @@ -61,8 +62,18 @@ def _conv3d(x, dilation = convert_to_list(dilation, dims, 'dilation') op_type = "conv3d" - return _C_ops.final_state_sparse_conv3d(x, weight, padding, dilation, - stride, groups, subm) + pre_bias = _C_ops.final_state_sparse_conv3d(x, weight, padding, dilation, + stride, groups, subm) + if bias is not None: + values = pre_bias.values() + add_bias = elementwise_add(values, bias, axis=1) + return sparse_coo_tensor( + pre_bias.indices(), + add_bias, + shape=pre_bias.shape, + stop_gradient=pre_bias.stop_gradient) + else: + return pre_bias def conv3d(x, diff --git a/python/paddle/sparse/functional/pooling.py b/python/paddle/sparse/functional/pooling.py new file mode 100644 index 0000000000000..ab5106b31689d --- /dev/null +++ b/python/paddle/sparse/functional/pooling.py @@ -0,0 +1,97 @@ +# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from ...fluid.layers import utils +from paddle import _C_ops, in_dynamic_mode +from paddle.nn.functional.pooling import _update_padding_nd + +__all__ = [] + + +def max_pool3d(x, + kernel_size, + stride=None, + padding=0, + ceil_mode=False, + data_format="NDHWC", + name=None): + """ + Implements sparse max pooling 3d operation. + See more details in :ref:`api_sparse_pooling_MaxPool3d` . + + Args: + x (Tensor): The input SparseCooTensor of pooling operator, which is a 5-D tensor with + shape [N, D, H, W, C]. The format of input tensor `"NDHWC"`, where N represents batch size, C represents the number of channels, D, H and W represent the depth, height and width of the feature respectively. + kernel_size (int|list|tuple): The pool kernel size. If the kernel size + is a tuple or list, it must contain three integers, + (kernel_size_Depth, kernel_size_Height, kernel_size_Width). + Otherwise, the pool kernel size will be the cube of an int. + stride (int|list|tuple): The pool stride size. If pool stride size is a tuple or list, + it must contain three integers, [stride_Depth, stride_Height, stride_Width). + Otherwise, the pool stride size will be a cube of an int. + padding (string|int|list|tuple): The padding size. Padding could be in one of the following forms. + 1. A string in ['valid', 'same']. + 2. An int, which means the feature map is zero padded by size of `padding` on every sides. + 3. A list[int] or tuple(int) whose length is 3, [pad_depth, pad_height, pad_weight] whose value means the padding size of each dimension. + 4. A list[int] or tuple(int) whose length is 6. [pad_depth_front, pad_depth_back, pad_height_top, pad_height_bottom, pad_width_left, pad_width_right] whose value means the padding size of each side. + 5. A list or tuple of pairs of integers. It has the form [[pad_before, pad_after], [pad_before, pad_after], ...]. Note that, the batch dimension and channel dimension should be [0,0] or (0,0). + The default value is 0. + ceil_mode (bool): ${ceil_mode_comment} + data_format (string): The data format of the input and output data. An optional string from: `"NCDHW"`, `"NDHWC"`. + The default is `"NCDHW"`. When it is `"NCDHW"`, the data is stored in the order of: + `[batch_size, input_channels, input_depth, input_height, input_width]`. Currently only support `"NDHWC"` . + name(str, optional): For detailed information, please refer + to :ref:`api_guide_Name`. Usually name is no need to set and + None by default. + + Returns: + Tensor: The output tensor of pooling result. The data type is same as input tensor. + + Examples: + .. code-block:: python + + import paddle + from paddle.fluid.framework import _test_eager_guard + + with _test_eager_guard(): + dense_x = paddle.randn((1, 4, 4, 4, 3)) + sparse_x = dense_x.to_sparse_coo(4) + kernel_sizes = [3, 3, 3] + paddings = [0, 0, 0] + strides = [1, 1, 1] + out = paddle.sparse.functional.max_pool3d(sparse_x, kernel_sizes, stride=strides, padding=paddings) + #[1, 2, 2, 2, 3] + """ + + assert in_dynamic_mode(), "Currently, Sparse API only support dynamic mode" + assert x.is_sparse_coo( + ), "Currently, sparse.relu only support the input of SparseCooTensor" + assert data_format == 'NDHWC', "Currently, sparse.max_pool3d only support data format of 'NDHWC'" + + kernel_size = utils.convert_to_list(kernel_size, 3, 'pool_size') + if stride is None: + stride = kernel_size + else: + stride = utils.convert_to_list(stride, 3, 'pool_stride') + + channel_last = True + + padding, padding_algorithm = _update_padding_nd( + padding, 3, channel_last=channel_last, ceil_mode=ceil_mode) + + #TODO(zkh2016): remove the dependency on dilation from the backend + dilation = [1, 1, 1] + + return _C_ops.final_state_sparse_maxpool(x, kernel_size, padding, dilation, + stride) diff --git a/python/paddle/sparse/layer/__init__.py b/python/paddle/sparse/layer/__init__.py index a0f9d068e677c..3a6d99392e4e8 100644 --- a/python/paddle/sparse/layer/__init__.py +++ b/python/paddle/sparse/layer/__init__.py @@ -13,7 +13,9 @@ # limitations under the License. from .activation import ReLU +from .norm import BatchNorm from .conv import Conv3D from .conv import SubmConv3D +from .pooling import MaxPool3D __all__ = [] diff --git a/python/paddle/sparse/layer/norm.py b/python/paddle/sparse/layer/norm.py new file mode 100644 index 0000000000000..83b738a5dc354 --- /dev/null +++ b/python/paddle/sparse/layer/norm.py @@ -0,0 +1,160 @@ +# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import paddle +import warnings + + +class BatchNorm(paddle.nn.BatchNorm1D): + r""" + Applies Batch Normalization over a SparseCooTensor as described in the paper Batch Normalization: Accelerating Deep Network Training by Reducing Internal Covariate Shift . + + When use_global_stats = False, the :math:`\mu_{\beta}` + and :math:`\sigma_{\beta}^{2}` are the statistics of one mini-batch. + Calculated as follows: + + .. math:: + + \mu_{\beta} &\gets \frac{1}{m} \sum_{i=1}^{m} x_i \qquad &//\ + \ mini-batch\ mean \\ + \sigma_{\beta}^{2} &\gets \frac{1}{m} \sum_{i=1}^{m}(x_i - \ + \mu_{\beta})^2 \qquad &//\ mini-batch\ variance \\ + + When use_global_stats = True, the :math:`\mu_{\beta}` + and :math:`\sigma_{\beta}^{2}` are not the statistics of one mini-batch. + They are global or running statistics (moving_mean and moving_variance). It usually got from the + pre-trained model. Calculated as follows: + + .. math:: + moving\_mean = moving\_mean * momentum + \mu_{\beta} * (1. - momentum) \quad &// global \ mean \\ + moving\_variance = moving\_variance * momentum + \sigma_{\beta}^{2} * (1. - momentum) \quad &// global \ variance \\ + + The normalization function formula is as follows: + + .. math:: + + \hat{x_i} &\gets \frac{x_i - \mu_\beta} {\sqrt{\sigma_{\beta}^{2} + \epsilon}} \qquad &//\ normalize \\ + y_i &\gets \gamma \hat{x_i} + \beta \qquad &//\ scale\ and\ shift + + - :math:`\epsilon` : add a smaller value to the variance to prevent division by zero + - :math:`\gamma` : trainable proportional parameter + - :math:`\beta` : trainable deviation parameter + + Parameters: + num_features(int): Indicate the number of channels of the input ``Tensor``. + momentum(float, optional): The value used for the moving_mean and moving_var computation. Default: 0.9. + epsilon(float, optional): The small value added to the variance to prevent division by zero. Default: 1e-5. + weight_attr(ParamAttr|bool, optional): The parameter attribute for Parameter `scale` + of batch_norm. If it is set to None or one attribute of ParamAttr, batch_norm + will create ParamAttr as weight_attr. If it is set to Fasle, the weight is not learnable. + If the Initializer of the weight_attr is not set, the parameter is initialized with Xavier. Default: None. + bias_attr(ParamAttr|bool, optional): The parameter attribute for the bias of batch_norm. + If it is set to None or one attribute of ParamAttr, batch_norm + will create ParamAttr as bias_attr. If it is set to Fasle, the weight is not learnable. + If the Initializer of the bias_attr is not set, the bias is initialized zero. Default: None. + data_format(str, optional): Specify the input data format, may be "NC", "NCL" or "NLC". Defalut "NCL". + use_global_stats(bool|None, optional): Whether to use global mean and variance. If set to False, use the statistics of one mini-batch, if set to True, use the global statistics, if set to None, use global statistics in the test phase and use the statistics of one mini-batch in the training phase. Default: None. + name(str, optional): Name for the BatchNorm, default is None. For more information, please refer to :ref:`api_guide_Name`.. + + Shape: + - x: A SparseCooTensor with layout = 'NDHWC'. + - output: SparseCooTensor with same shape as input x. + + Returns: + None. + + + Examples: + .. code-block:: python + + import paddle + from paddle.fluid.framework import _test_eager_guard + + with _test_eager_guard(): + paddle.seed(123) + channels = 3 + x_data = paddle.randn((1, 6, 6, 6, channels)).astype('float32') + dense_x = paddle.to_tensor(x_data) + sparse_x = dense_x.to_sparse_coo(4) + batch_norm = paddle.sparse.BatchNorm(channels) + batch_norm_out = batch_norm(sparse_x) + print(batch_norm_out.shape) + # [1, 6, 6, 6, 3] + """ + + def __init__(self, + num_features, + momentum=0.9, + epsilon=1e-05, + weight_attr=None, + bias_attr=None, + data_format='NDHWC', + use_global_stats=None, + name=None): + super(BatchNorm, self).__init__( + num_features, + momentum=momentum, + epsilon=epsilon, + weight_attr=weight_attr, + bias_attr=bias_attr, + data_format=data_format, + use_global_stats=use_global_stats, + name=name) + + def _check_data_format(self, input): + if input != "NDHWC": + raise ValueError('sparse BatchNorm only support layout of "NDHWC"') + + def forward(self, input): + values = input.values() + self._check_data_format(self._data_format) + + if len(values.shape) != 2: + raise ValueError('expected 2D input.values() (got {}D)'.format( + len(values.shape))) + + if self.training: + warnings.warn( + "When training, we now always track global mean and variance.") + + batch_norm_out = paddle.nn.functional.batch_norm( + values, + self._mean, + self._variance, + weight=self.weight, + bias=self.bias, + training=self.training, + momentum=self._momentum, + epsilon=self._epsilon, + data_format='NC', + use_global_stats=self._use_global_stats) + + return paddle.sparse.sparse_coo_tensor( + input.indices(), + batch_norm_out, + shape=input.shape, + stop_gradient=input.stop_gradient) diff --git a/python/paddle/sparse/layer/pooling.py b/python/paddle/sparse/layer/pooling.py new file mode 100644 index 0000000000000..9cfe463eed577 --- /dev/null +++ b/python/paddle/sparse/layer/pooling.py @@ -0,0 +1,105 @@ +# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from paddle.nn import Layer +from .. import functional as F + + +class MaxPool3D(Layer): + """ + This operation applies 3D max pooling over input features based on the sparse input, + and kernel_size, stride, padding parameters. Input(X) and Output(Out) are + in NDHWC format, where N is batch size, C is the number of channels, + H is the height of the feature, D is the depth of the feature, and W is the width of the feature. + + Parameters: + kernel_size(int|list|tuple): The pool kernel size. If the kernel size + is a tuple or list, it must contain three integers, + (kernel_size_Depth, kernel_size_Height, kernel_size_Width). + Otherwise, the pool kernel size will be the cube of an int. + stride(int|list|tuple, optional): The pool stride size. If pool stride size is a tuple or list, + it must contain three integers, [stride_Depth, stride_Height, stride_Width). + Otherwise, the pool stride size will be a cube of an int. + Default None, then stride will be equal to the kernel_size. + padding(str|int|list|tuple, optional): The padding size. Padding could be in one of the following forms. + 1. A string in ['valid', 'same']. + 2. An int, which means the feature map is zero padded by size of `padding` on every sides. + 3. A list[int] or tuple(int) whose length is 3, [pad_depth, pad_height, pad_weight] whose value means the padding size of each dimension. + 4. A list[int] or tuple(int) whose length is \6. [pad_depth_front, pad_depth_back, pad_height_top, pad_height_bottom, pad_width_left, pad_width_right] whose value means the padding size of each side. + 5. A list or tuple of pairs of integers. It has the form [[pad_before, pad_after], [pad_before, pad_after], ...]. Note that, the batch dimension and channel dimension should be [0,0] or (0,0). + The default value is 0. + ceil_mode(bool, optional): ${ceil_mode_comment} + return_mask(bool, optional): Whether to return the max indices along with the outputs. + data_format(str, optional): The data format of the input and output data. An optional string from: `"NCDHW"`, + `"NDHWC"`. The default is `"NCDHW"`. When it is `"NCDHW"`, the data is stored in the order of: + `[batch_size, input_channels, input_depth, input_height, input_width]`. Currently, only support "NDHWC". + name(str, optional): For detailed information, please refer to :ref:`api_guide_Name`. + Usually name is no need to set and None by default. + + + Returns: + A callable object of MaxPool3D. + + Shape: + - x(Tensor): The input SparseCooTensor of max pool3d operator, which is a 5-D tensor. + The data type can be float32, float64. + - output(Tensor): The output tensor of max pool3d operator, which is a 5-D tensor. + The data type is same as input x. + + Examples: + .. code-block:: python + + import paddle + from paddle.fluid.framework import _test_eager_guard + + with _test_eager_guard(): + dense_x = paddle.randn((2, 3, 6, 6, 3)) + sparse_x = dense_x.to_sparse_coo(4) + max_pool3d = paddle.sparse.MaxPool3D( + kernel_size=3, data_format='NDHWC') + out = max_pool3d(sparse_x) + #shape=[2, 1, 2, 2, 3] + + """ + + def __init__(self, + kernel_size, + stride=None, + padding=0, + return_mask=False, + ceil_mode=False, + data_format="NDHWC", + name=None): + super(MaxPool3D, self).__init__() + self.ksize = kernel_size + self.stride = stride + self.padding = padding + self.return_mask = return_mask + self.ceil_mode = ceil_mode + self.data_format = data_format + self.name = name + + def forward(self, x): + return F.max_pool3d( + x, + kernel_size=self.ksize, + stride=self.stride, + padding=self.padding, + ceil_mode=self.ceil_mode, + data_format=self.data_format, + name=self.name) + + def extra_repr(self): + return 'kernel_size={ksize}, stride={stride}, padding={padding}'.format( + **self.__dict__) diff --git a/python/paddle/tensor/creation.py b/python/paddle/tensor/creation.py index aeec256bc1580..a5a4df6571b77 100644 --- a/python/paddle/tensor/creation.py +++ b/python/paddle/tensor/creation.py @@ -827,6 +827,11 @@ def arange(start=0, end=None, step=1, dtype=None, name=None): end = start start = 0 + out_shape = None + if not isinstance(start, Variable) and not isinstance( + end, Variable) and not isinstance(step, Variable): + out_shape = [int(math.ceil((end - start) / step))] + if not isinstance(dtype, core.VarDesc.VarType): dtype = convert_np_dtype_to_dtype_(dtype) @@ -857,11 +862,6 @@ def arange(start=0, end=None, step=1, dtype=None, name=None): out.stop_gradient = True return out - out_shape = None - if not isinstance(start, Variable) and not isinstance( - end, Variable) and not isinstance(step, Variable): - out_shape = [int(math.ceil((end - start) / step))] - check_dtype(dtype, 'dtype', ['float32', 'float64', 'int32', 'int64'], 'range/arange') helper = LayerHelper('range', **locals()) @@ -873,6 +873,8 @@ def arange(start=0, end=None, step=1, dtype=None, name=None): 'Step': step}, outputs={'Out': out}) out.stop_gradient = True + if out_shape is not None: + out.desc.set_shape(out_shape) return out diff --git a/python/paddle/tensor/einsum.py b/python/paddle/tensor/einsum.py index 06c2a82fd696d..dd11477532d24 100644 --- a/python/paddle/tensor/einsum.py +++ b/python/paddle/tensor/einsum.py @@ -20,6 +20,10 @@ from .manipulation import squeeze, unsqueeze, reshape from .math import multiply from .math import sum as paddle_sum +from ..fluid.framework import _in_legacy_dygraph +from paddle import _C_ops +from ..fluid.data_feeder import check_variable_and_dtype, check_type, check_dtype +from ..fluid.layer_helper import LayerHelper from paddle.common_ops_import import dygraph_only @@ -660,6 +664,26 @@ def plan_einsum(operands, g_view, g_shape, g_supports, g_count, n_bcast): return plan +def einsum_v2(equation, *operands): + if _in_legacy_dygraph(): + # dygraph + return _C_ops.einsum(operands, 'equation', equation) + # static graph + for inp in operands: + check_variable_and_dtype(inp, 'dtype', ['float32', 'float64'], 'einsum') + check_type(equation, 'equation', str, 'einsum') + helper = LayerHelper('einsum', **locals()) + out = helper.create_variable_for_type_inference(dtype=operands[0].dtype) + attrs = dict() + attrs['equation'] = equation + helper.append_op( + type='einsum', + inputs={'Operands': operands}, + outputs={'Out': out}, + attrs=attrs, ) + return out + + def einsum(equation, *operands): r""" einsum(equation, *operands) @@ -817,6 +841,9 @@ def einsum(equation, *operands): # [0.50226176, 0.24512935, 0.39881429], # [0.51476848, 0.23367381, 0.39229113]]]) """ + import os + if int(os.environ.get('FLAGS_new_einsum', "0")): + return einsum_v2(equation, *operands) nop = len(operands) assert nop > 0, "At least one operand is expected." diff --git a/python/paddle/tensor/manipulation.py b/python/paddle/tensor/manipulation.py index b0e0082c6d9c4..127aa71137dff 100755 --- a/python/paddle/tensor/manipulation.py +++ b/python/paddle/tensor/manipulation.py @@ -45,9 +45,9 @@ def cast(x, dtype): equals the input dtype, but it's fine if you do so. Args: - x(Tensor): An input N-D Tensor with data type bool, float16, + x (Tensor): An input N-D Tensor with data type bool, float16, float32, float64, int32, int64, uint8. - dtype(np.dtype|str): Data type of the output: + dtype (np.dtype|str): Data type of the output: bool, float16, float32, float64, int8, int32, int64, uint8. Returns: @@ -601,8 +601,7 @@ def crop(x, shape=None, offsets=None, name=None): Tensor. When it is a list, each element can be an integer or a Tensor of shape: [1]. If Variable contained, it is suitable for the case that the offsets may be changed each iteration. Default: None, the offsets are 0 at each dimension. - name(str, optional): The default value is None. Normally there is no need for user to set - this property. For more information, please refer to :ref:`api_guide_Name` . + name (str, optional): Name for the operation (optional, default is None). For more information, please refer to :ref:`api_guide_Name`. Returns: Tensor: The cropped Tensor has same data type with `x`. @@ -742,8 +741,8 @@ def fill_(x, value): This function fill the Tensor with value inplace. Args: - x(Tensor): ``x`` is the Tensor we want to filled data inplace - value(Scale): ``value`` is the value to be filled in x + x (Tensor): ``x`` is the Tensor we want to filled data inplace + value (Scale): ``value`` is the value to be filled in x Returns: x(Tensor): Tensor x filled with value inplace @@ -776,10 +775,10 @@ def zero_(x): This function fill the Tensor with zero inplace. Args: - x(Tensor): ``x`` is the Tensor we want to filled with zero inplace + x (Tensor): ``x`` is the Tensor we want to filled with zero inplace Returns: - x(Tensor): Tensor x filled with zero inplace + x (Tensor): Tensor x filled with zero inplace Examples: .. code-block:: python @@ -798,19 +797,21 @@ def zero_(x): @dygraph_only def fill_diagonal_(x, value, offset=0, wrap=False, name=None): """ - **Notes**: - **This API is ONLY available in Dygraph mode** + Note: + This API is ONLY available in Dygraph mode. + This function fill the value into the x Tensor's diagonal inplace. + Args: x(Tensor): ``x`` is the original Tensor value(Scale): ``value`` is the value to filled in x offset(int,optional): the offset to the main diagonal. Default: 0 (main diagonal). wrap(bool,optional): the diagonal 'wrapped' after N columns for tall matrices. name(str,optional): Name for the operation (optional, default is None) + Returns: Tensor: Tensor with diagonal filled with value. - Returns type: - dtype is same as x Tensor + Examples: .. code-block:: python import paddle @@ -874,25 +875,22 @@ def _fill_diagonal_tensor_impl(x, y, offset=0, dim1=0, dim2=1, inplace=False): def fill_diagonal_tensor_(x, y, offset=0, dim1=0, dim2=1, name=None): """ - **Notes**: - **This API is ONLY available in Dygraph mode** + Note: + This API is ONLY available in Dygraph mode. This function fill the source Tensor y into the x Tensor's diagonal inplace. Args: - x(Tensor): ``x`` is the original Tensor - y(Tensor): ``y`` is the Tensor to filled in x - dim1(int,optional): first dimension with respect to which to fill diagonal. Default: 0. - dim2(int,optional): second dimension with respect to which to fill diagonal. Default: 1. - offset(int,optional): the offset to the main diagonal. Default: 0 (main diagonal). - name(str,optional): Name for the operation (optional, default is None) + x (Tensor): ``x`` is the original Tensor + y (Tensor): ``y`` is the Tensor to filled in x + dim1 (int,optional): first dimension with respect to which to fill diagonal. Default: 0. + dim2 (int,optional): second dimension with respect to which to fill diagonal. Default: 1. + offset (int,optional): the offset to the main diagonal. Default: 0 (main diagonal). + name (str, optional): Name for the operation (optional, default is None). For more information, please refer to :ref:`api_guide_Name`. Returns: Tensor: Tensor with diagonal filled with y. - Returns type: - list: dtype is same as x Tensor - Examples: .. code-block:: python @@ -913,19 +911,16 @@ def fill_diagonal_tensor(x, y, offset=0, dim1=0, dim2=1, name=None): This function fill the source Tensor y into the x Tensor's diagonal. Args: - x(Tensor): ``x`` is the original Tensor - y(Tensor): ``y`` is the Tensor to filled in x - dim1(int,optional): first dimension with respect to which to fill diagonal. Default: 0. - dim2(int,optional): second dimension with respect to which to fill diagonal. Default: 1. - offset(int,optional): the offset to the main diagonal. Default: 0 (main diagonal). - name(str,optional): Name for the operation (optional, default is None) + x (Tensor): ``x`` is the original Tensor + y (Tensor): ``y`` is the Tensor to filled in x + dim1 (int,optional): first dimension with respect to which to fill diagonal. Default: 0. + dim2 (int,optional): second dimension with respect to which to fill diagonal. Default: 1. + offset (int,optional): the offset to the main diagonal. Default: 0 (main diagonal). + name (str, optional): Name for the operation (optional, default is None). For more information, please refer to :ref:`api_guide_Name`. Returns: Tensor: Tensor with diagonal filled with y. - Returns type: - list: dtype is same as x Tensor - Examples: .. code-block:: python @@ -944,19 +939,17 @@ def fill_diagonal_tensor(x, y, offset=0, dim1=0, dim2=1, name=None): @dygraph_only def tolist(x): """ - **Notes**: - **This API is ONLY available in Dygraph mode** + Note: + This API is ONLY available in Dygraph mode. This function translate the paddle.Tensor to python list. Args: - x(Tensor): ``x`` is the Tensor we want to translate to list + x (Tensor): ``x`` is the Tensor we want to translate to list. Returns: list: A list that contain the same value of current Tensor. - Returns type: - list: dtype is same as current Tensor Examples: .. code-block:: python @@ -980,15 +973,13 @@ def concat(x, axis=0, name=None): This OP concatenates the input along the axis. Args: - x(list|tuple): ``x`` is a Tensor list or Tensor tuple which is with data type bool, float16, + x (list|tuple): ``x`` is a Tensor list or Tensor tuple which is with data type bool, float16, float32, float64, int32, int64, uint8. All the Tensors in ``x`` must have same data type. - axis(int|Tensor, optional): Specify the axis to operate on the input Tensors. + axis (int|Tensor, optional): Specify the axis to operate on the input Tensors. It's a scalar with data type int or a Tensor with shape [1] and data type int32 or int64. The effective range is [-R, R), where R is Rank(x). When ``axis < 0``, it works the same way as ``axis+R``. Default is 0. - name (str, optional): The default value is None. Normally there is no - need for user to set this property. For more information, please - refer to :ref:`api_guide_Name`. + name (str, optional): Name for the operation (optional, default is None). For more information, please refer to :ref:`api_guide_Name`. Returns: Tensor: A Tensor with the same data type as ``x``. @@ -1097,12 +1088,10 @@ def broadcast_tensors(input, name=None): If you want know more about broadcasting, please refer to :ref:`user_guide_broadcasting`. Args: - input(list|tuple): ``input`` is a Tensor list or Tensor tuple which is with data type bool, + input (list|tuple): ``input`` is a Tensor list or Tensor tuple which is with data type bool, float16, float32, float64, int32, int64. All the Tensors in ``input`` must have same data type. Currently we only support tensors with rank no greater than 5. - - name (str, optional): The default value is None. Normally there is no need for user to set this property. - For more information, please refer to :ref:`api_guide_Name`. + name (str, optional): Name for the operation (optional, default is None). For more information, please refer to :ref:`api_guide_Name`. Returns: list(Tensor): The list of broadcasted tensors following the same order as ``input``. @@ -1192,8 +1181,7 @@ def flip(x, axis, name=None): x (Tensor): A Tensor(or LoDTensor) with shape :math:`[N_1, N_2,..., N_k]` . The data type of the input Tensor x should be float32, float64, int32, int64, bool. axis (list|tuple|int): The axis(axes) to flip on. Negative indices for indexing from the end are accepted. - name (str, optional): The default value is None. Normally there is no need for user to set this property. - For more information, please refer to :ref:`api_guide_Name` . + name (str, optional): Name for the operation (optional, default is None). For more information, please refer to :ref:`api_guide_Name`. Returns: Tensor: Tensor or LoDTensor calculated by flip layer. The data type is same with input x. @@ -3143,20 +3131,19 @@ def reshape(x, shape, name=None): the corresponding dimension of x. Args: - x(Tensor): An N-D Tensor. The data type is ``float32``, ``float64``, ``int32``, ``int64`` or ``bool`` - shape(list|tuple|Tensor): Define the target shape. At most one dimension of the target shape can be -1. + x (Tensor): An N-D Tensor. The data type is ``float32``, ``float64``, ``int32``, ``int64`` or ``bool`` + shape (list|tuple|Tensor): Define the target shape. At most one dimension of the target shape can be -1. The data type is ``int32`` . If ``shape`` is a list or tuple, the elements of it should be integers or Tensors with shape [1]. If ``shape`` is an Tensor, it should be an 1-D Tensor . - name(str, optional): The default value is None. Normally there is no need for user to set this property. - For more information, please refer to :ref:`api_guide_Name` . + name (str, optional): Name for the operation (optional, default is None). For more information, please refer to :ref:`api_guide_Name`. Returns: Tensor: A reshaped Tensor with the same data type as ``x``. Examples: .. code-block:: python + :name: code-example1 - import numpy as np import paddle x = paddle.rand([2, 4, 6], dtype="float32") @@ -3170,9 +3157,9 @@ def reshape(x, shape, name=None): print(out) # the shape of out_2 is [4, 12]. - shape_tensor = paddle.to_tensor(np.array([8, 6]).astype("int32")) + shape_tensor = paddle.to_tensor([8, 6], dtype=paddle.int32) out = paddle.reshape(x, shape=shape_tensor) - print(out) + print(out.shape) # the shape is [8, 6]. # out shares data with x in dygraph mode x[0, 0, 0] = 10. @@ -4113,14 +4100,12 @@ def take_along_axis(arr, indices, axis): Examples: .. code-block:: python + :name: code-example1 import paddle - import numpy as np - x_np = np.array([[1, 2, 3], [4, 5, 6], [7,8,9]]) - index_np = np.array([[0]]) - x = paddle.to_tensor(x_np) - index = paddle.to_tensor(index_np) + x = paddle.to_tensor([[1, 2, 3], [4, 5, 6], [7,8,9]]) + index = paddle.to_tensor([[0]]) axis = 0 result = paddle.take_along_axis(x, index, axis) print(result) @@ -4180,14 +4165,12 @@ def put_along_axis(arr, indices, values, axis, reduce='assign'): Examples: .. code-block:: python + :name: code-example1 import paddle - import numpy as np - x_np = np.array([[10, 30, 20], [60, 40, 50]]) - index_np = np.array([[0]]) - x = paddle.to_tensor(x_np) - index = paddle.to_tensor(index_np) + x = paddle.to_tensor([[10, 30, 20], [60, 40, 50]]) + index = paddle.to_tensor([[0]]) value = 99 axis = 0 result = paddle.put_along_axis(x, index, value, axis) diff --git a/python/paddle/tensor/math.py b/python/paddle/tensor/math.py index 7e0b2e5424dad..b7b08af9e60bc 100644 --- a/python/paddle/tensor/math.py +++ b/python/paddle/tensor/math.py @@ -146,12 +146,12 @@ def scale(x, scale=1.0, bias=0.0, bias_after_scale=True, act=None, name=None): Out=scale*(X+bias) Args: - x(Tensor): Input N-D Tensor of scale operator. Data type can be float32, float64, int8, int16, int32, int64, uint8. - scale(float|Tensor): The scale factor of the input, it should be a float number or a Tensor with shape [1] and data type as float32. - bias(float): The bias to be put on the input. - bias_after_scale(bool): Apply bias addition after or before scaling. It is useful for numeric stability in some circumstances. - act(str, optional): Activation applied to the output such as tanh, softmax, sigmoid, relu. - name(str, optional): The default value is None. Normally there is no need for user to set this property. For more information, please refer to :ref:`api_guide_Name` + x (Tensor): Input N-D Tensor of scale operator. Data type can be float32, float64, int8, int16, int32, int64, uint8. + scale (float|Tensor): The scale factor of the input, it should be a float number or a Tensor with shape [1] and data type as float32. + bias (float): The bias to be put on the input. + bias_after_scale (bool): Apply bias addition after or before scaling. It is useful for numeric stability in some circumstances. + act (str, optional): Activation applied to the output such as tanh, softmax, sigmoid, relu. + name (str, optional): Name for the operation (optional, default is None). For more information, please refer to :ref:`api_guide_Name`. Returns: Tensor: Output tensor of scale operator, with shape and data type same as input. @@ -281,24 +281,23 @@ def multiplex(inputs, index, name=None): Args: inputs (list): The input Tensor list. The list elements are N-D Tensors of data types float32, float64, int32, int64. All input Tensor shapes should be the same and rank must be at least 2. index (Tensor): Used to select some rows in the input Tensor to construct an index of the output Tensor. It is a 2-D Tensor with data type int32 or int64 and shape [M, 1], where M is the number of input Tensors. - name(str, optional): The default value is None. Normally there is no - need for user to set this property. For more information, please - refer to :ref:`api_guide_Name`. + name (str, optional): Name for the operation (optional, default is None). For more information, please refer to :ref:`api_guide_Name`. + Returns: Tensor: Output of multiplex OP, with data type being float32, float64, int32, int64. Examples: .. code-block:: python + :name: code-example1 import paddle - import numpy as np - img1 = np.array([[1, 2], [3, 4]]).astype(np.float32) - img2 = np.array([[5, 6], [7, 8]]).astype(np.float32) - inputs = [paddle.to_tensor(img1), paddle.to_tensor(img2)] - index = paddle.to_tensor(np.array([[1], [0]]).astype(np.int32)) + img1 = paddle.to_tensor([[1, 2], [3, 4]], dtype=paddle.float32) + img2 = paddle.to_tensor([[5, 6], [7, 8]], dtype=paddle.float32) + inputs = [img1, img2] + index = paddle.to_tensor([[1], [0]], dtype=paddle.int32) res = paddle.multiplex(inputs, index) - print(res) # [array([[5., 6.], [3., 4.]], dtype=float32)] + print(res) # Tensor([[5., 6.], [3., 4.]], dtype=float32) """ if _non_static_mode(): @@ -1077,8 +1076,7 @@ def sum(x, axis=None, dtype=None, keepdim=False, name=None): output Tensor. The result Tensor will have one fewer dimension than the :attr:`x` unless :attr:`keepdim` is true, default value is False. - name (str, optional): The default value is None. Normally there is no need for - user to set this property. For more information, please refer to :ref:`api_guide_Name` + name (str, optional): Name for the operation (optional, default is None). For more information, please refer to :ref:`api_guide_Name`. Returns: Tensor: Results of summation operation on the specified axis of input Tensor `x`, @@ -1134,15 +1132,10 @@ def sum(x, axis=None, dtype=None, keepdim=False, name=None): else: reduce_all_flag = False - def get_dtype(x, dtype): - if dtype is not None: - return (True, dtype) - src_type = convert_dtype(x.dtype) - if src_type in ['bool','int32', 'int64']: - return (True, 'int64') - return (False, src_type) - - dtype_flag, dtype = get_dtype(x, dtype) + dtype_flag = False + if dtype is not None: + dtype_flag = True + dtype = convert_np_dtype_to_dtype_(dtype) if in_dygraph_mode(): if reduce_all_flag: @@ -1150,17 +1143,14 @@ def get_dtype(x, dtype): else: axis = axis if axis != None and axis != [] else [0] - out_dtype = convert_np_dtype_to_dtype_(dtype) - out = _C_ops.final_state_sum(x, axis, out_dtype, keepdim) - return out + return _C_ops.final_state_sum(x, axis, dtype, keepdim) if _in_legacy_dygraph(): axis = axis if axis != None and axis != [] else [0] if dtype_flag: return _C_ops.reduce_sum(x, 'dim', axis, 'keep_dim', keepdim, 'reduce_all', reduce_all_flag, 'in_dtype', - x.dtype, 'out_dtype', - convert_np_dtype_to_dtype_(dtype)) + x.dtype, 'out_dtype', dtype) else: return _C_ops.reduce_sum(x, 'dim', axis, 'keep_dim', keepdim, 'reduce_all', reduce_all_flag) @@ -1174,7 +1164,7 @@ def get_dtype(x, dtype): if dtype_flag: attrs.update({ 'in_dtype': x.dtype, - 'out_dtype': convert_np_dtype_to_dtype_(dtype) + 'out_dtype': dtype }) check_variable_and_dtype( @@ -1188,7 +1178,7 @@ def get_dtype(x, dtype): helper = LayerHelper('sum', **locals()) if dtype_flag: out = helper.create_variable_for_type_inference( - dtype=convert_np_dtype_to_dtype_(dtype)) + dtype=dtype) else: out = helper.create_variable_for_type_inference(dtype=x.dtype) helper.append_op( @@ -1216,8 +1206,7 @@ def nansum(x, axis=None, dtype=None, keepdim=False, name=None): output Tensor. The result Tensor will have one fewer dimension than the :attr:`x` unless :attr:`keepdim` is true, default value is False. - name (str, optional): The default value is None. Normally there is no need for - user to set this property. For more information, please refer to :ref:`api_guide_Name` + name (str, optional): Name for the operation (optional, default is None). For more information, please refer to :ref:`api_guide_Name`. Returns: Tensor: Results of summation operation on the specified axis of input Tensor `x`, @@ -1368,8 +1357,7 @@ def add_n(inputs, name=None): Args: inputs (Tensor|list[Tensor]|tuple[Tensor]): A Tensor or a list/tuple of Tensors. The shape and data type of the list/tuple elements should be consistent. Input can be multi-dimensional Tensor, and data types can be: float32, float64, int32, int64. - name(str, optional): The default value is None. Normally there is no need for - user to set this property. For more information, please refer to :ref:`api_guide_Name` + name (str, optional): Name for the operation (optional, default is None). For more information, please refer to :ref:`api_guide_Name`. Returns: Tensor, the sum of input :math:`inputs` , its shape and data types are consistent with :math:`inputs`. @@ -1480,8 +1468,7 @@ def mm(input, mat2, name=None): Args: input (Tensor): The input tensor which is a Tensor. mat2 (Tensor): The input tensor which is a Tensor. - name(str, optional): The default value is None. Normally there is no need for - user to set this property. For more information, please refer to :ref:`api_guide_Name` + name (str, optional): Name for the operation (optional, default is None). For more information, please refer to :ref:`api_guide_Name`. Returns: Tensor: The product Tensor. @@ -1599,7 +1586,7 @@ def addmm(input, x, y, beta=1.0, alpha=1.0, name=None): y (Tensor): The second input Tensor for matrix multiplication. beta (float): Coefficient of $input$. alpha (float): Coefficient of $x*y$. - name (str, optional): Name of the output. Normally there is no need for user to set this property. For more information, please refer to :ref:`api_guide_Name`. Default is None. + name (str, optional): Name for the operation (optional, default is None). For more information, please refer to :ref:`api_guide_Name`. Returns: Tensor: The output Tensor of addmm op. @@ -1727,8 +1714,7 @@ def inner(x, y, name=None): Args: x (Tensor): An N-D Tensor or a Scalar Tensor. If its not a scalar Tensor, its last dimensions must match y's. y (Tensor): An N-D Tensor or a Scalar Tensor. If its not a scalar Tensor, its last dimensions must match x's. - name(str, optional): The default value is None. Normally there is no need for - user to set this property. For more information, please refer to :ref:`api_guide_Name` + name (str, optional): Name for the operation (optional, default is None). For more information, please refer to :ref:`api_guide_Name`. Returns: Tensor: The inner-product Tensor, the output shape is x.shape[:-1] + y.shape[:-1]. @@ -1799,8 +1785,7 @@ def outer(x, y, name=None): Args: x (Tensor): An N-D Tensor or a Scalar Tensor. y (Tensor): An N-D Tensor or a Scalar Tensor. - name(str, optional): The default value is None. Normally there is no need for - user to set this property. For more information, please refer to :ref:`api_guide_Name` + name (str, optional): Name for the operation (optional, default is None). For more information, please refer to :ref:`api_guide_Name`. Returns: Tensor: The outer-product Tensor. @@ -1923,9 +1908,7 @@ def inverse(x, name=None): dimensions should be equal. When the number of dimensions is greater than 2, it is treated as batches of square matrix. The data type can be float32 and float64. - name (str, optional): The default value is None. Normally there is no need for - user to set this property. For more information, - please refer to :ref:`api_guide_Name` + name (str, optional): Name for the operation (optional, default is None). For more information, please refer to :ref:`api_guide_Name`. Returns: Tensor: A Tensor holds the inverse of x. The shape and data type @@ -1989,18 +1972,17 @@ def max(x, axis=None, keepdim=False, name=None): Args: - x(Tensor): A tensor, the data type is float32, float64, int32, int64. - axis(int|list|tuple, optional): The axis along which the maximum is computed. + x (Tensor): A tensor, the data type is float32, float64, int32, int64. + axis (int|list|tuple, optional): The axis along which the maximum is computed. If :attr:`None`, compute the maximum over all elements of `x` and return a Tensor with a single element, otherwise must be in the range :math:`[-x.ndim(x), x.ndim(x))`. If :math:`axis[i] < 0`, the axis to reduce is :math:`x.ndim + axis[i]`. - keepdim(bool, optional): Whether to reserve the reduced dimension in the + keepdim (bool, optional): Whether to reserve the reduced dimension in the output Tensor. The result tensor will have one fewer dimension than the `x` unless :attr:`keepdim` is true, default value is False. - name(str, optional): The default value is None. Normally there is no need for - user to set this property. For more information, please refer to :ref:`api_guide_Name` + name (str, optional): Name for the operation (optional, default is None). For more information, please refer to :ref:`api_guide_Name`. Returns: Tensor, results of maximum on the specified axis of input tensor, @@ -2093,18 +2075,17 @@ def min(x, axis=None, keepdim=False, name=None): while min propagates gradient to all of them. Args: - x(Tensor): A tensor, the data type is float32, float64, int32, int64. - axis(int|list|tuple, optional): The axis along which the minimum is computed. + x (Tensor): A tensor, the data type is float32, float64, int32, int64. + axis (int|list|tuple, optional): The axis along which the minimum is computed. If :attr:`None`, compute the minimum over all elements of `x` and return a Tensor with a single element, otherwise must be in the range :math:`[-x.ndim, x.ndim)`. If :math:`axis[i] < 0`, the axis to reduce is :math:`x.ndim + axis[i]`. - keepdim(bool, optional): Whether to reserve the reduced dimension in the + keepdim (bool, optional): Whether to reserve the reduced dimension in the output Tensor. The result tensor will have one fewer dimension than the `x` unless :attr:`keepdim` is true, default value is False. - name(str, optional): The default value is None. Normally there is no need for - user to set this property. For more information, please refer to :ref:`api_guide_Name` + name (str, optional): Name for the operation (optional, default is None). For more information, please refer to :ref:`api_guide_Name`. Returns: Tensor, results of minimum on the specified axis of input tensor, @@ -2197,19 +2178,18 @@ def amax(x, axis=None, keepdim=False, name=None): while max propagates gradient to all of them. Args: - x(Tensor): A tensor, the data type is float32, float64, int32, int64, + x (Tensor): A tensor, the data type is float32, float64, int32, int64, the dimension is no more than 4. - axis(int|list|tuple, optional): The axis along which the maximum is computed. + axis (int|list|tuple, optional): The axis along which the maximum is computed. If :attr:`None`, compute the maximum over all elements of `x` and return a Tensor with a single element, otherwise must be in the range :math:`[-x.ndim(x), x.ndim(x))`. If :math:`axis[i] < 0`, the axis to reduce is :math:`x.ndim + axis[i]`. - keepdim(bool, optional): Whether to reserve the reduced dimension in the + keepdim (bool, optional): Whether to reserve the reduced dimension in the output Tensor. The result tensor will have one fewer dimension than the `x` unless :attr:`keepdim` is true, default value is False. - name(str, optional): The default value is None. Normally there is no need for - user to set this property. For more information, please refer to :ref:`api_guide_Name` + name (str, optional): Name for the operation (optional, default is None). For more information, please refer to :ref:`api_guide_Name`. Returns: Tensor, results of maximum on the specified axis of input tensor, @@ -2310,19 +2290,18 @@ def amin(x, axis=None, keepdim=False, name=None): while min propagates gradient to all of them. Args: - x(Tensor): A tensor, the data type is float32, float64, int32, int64, + x (Tensor): A tensor, the data type is float32, float64, int32, int64, the dimension is no more than 4. - axis(int|list|tuple, optional): The axis along which the minimum is computed. + axis (int|list|tuple, optional): The axis along which the minimum is computed. If :attr:`None`, compute the minimum over all elements of `x` and return a Tensor with a single element, otherwise must be in the range :math:`[-x.ndim, x.ndim)`. If :math:`axis[i] < 0`, the axis to reduce is :math:`x.ndim + axis[i]`. - keepdim(bool, optional): Whether to reserve the reduced dimension in the + keepdim (bool, optional): Whether to reserve the reduced dimension in the output Tensor. The result tensor will have one fewer dimension than the `x` unless :attr:`keepdim` is true, default value is False. - name(str, optional): The default value is None. Normally there is no need for - user to set this property. For more information, please refer to :ref:`api_guide_Name` + name (str, optional): Name for the operation (optional, default is None). For more information, please refer to :ref:`api_guide_Name`. Returns: Tensor, results of minimum on the specified axis of input tensor, @@ -2421,8 +2400,8 @@ def log1p(x, name=None): Args: x (Tensor): Input Tensor. Must be one of the following types: float32, float64. - name(str, optional): The default value is None. Normally there is no need for - user to set this property. For more information, please refer to :ref:`api_guide_Name` + name (str, optional): Name for the operation (optional, default is None). For more information, please refer to :ref:`api_guide_Name`. + Returns: Tensor, the natural log of the input Tensor computed element-wise. @@ -2459,7 +2438,7 @@ def log2(x, name=None): Args: x (Tensor): Input tensor must be one of the following types: float32, float64. - name (str|None): The default value is None. Normally there is no need for user to set this property. For more information, please refer to :ref:`api_guide_Name` + name (str, optional): Name for the operation (optional, default is None). For more information, please refer to :ref:`api_guide_Name`. Returns: @@ -2511,7 +2490,7 @@ def log10(x, name=None): Args: x (Tensor): Input tensor must be one of the following types: float32, float64. - name (str|None): The default value is None. Normally there is no need for user to set this property. For more information, please refer to :ref:`api_guide_Name` + name (str, optional): Name for the operation (optional, default is None). For more information, please refer to :ref:`api_guide_Name`. Returns: @@ -2568,9 +2547,7 @@ def clip(x, min=None, max=None, name=None): with shape [1] and type ``int32``, ``float32``, ``float64``. max (float|int|Tensor): The upper bound with type ``float``, ``int`` or a ``Tensor`` with shape [1] and type ``int32``, ``float32``, ``float64``. - name (str, optional): The default value is None. Normally there is no - need for user to set this property. For more information, please - refer to :ref:`api_guide_Name`. + name (str, optional): Name for the operation (optional, default is None). For more information, please refer to :ref:`api_guide_Name`. Returns: Tensor: A Tensor with the same data type and data shape as input. @@ -2700,11 +2677,11 @@ def trace(x, offset=0, axis1=0, axis2=1, name=None): - Note that if offset is out of input's shape indicated by axis1 and axis2, 0 will be returned. Args: - x(Tensor): The input tensor x. Must be at least 2-dimensional. The input data type should be float32, float64, int32, int64. - offset(int, optional): Which diagonals in input tensor x will be taken. Default: 0 (main diagonals). - axis1(int, optional): The first axis with respect to take diagonal. Default: 0. - axis2(int, optional): The second axis with respect to take diagonal. Default: 1. - name (str, optional): Normally there is no need for user to set this property. For more information, please refer to :ref:`api_guide_Name`. Default: None. + x (Tensor): The input tensor x. Must be at least 2-dimensional. The input data type should be float32, float64, int32, int64. + offset (int, optional): Which diagonals in input tensor x will be taken. Default: 0 (main diagonals). + axis1 (int, optional): The first axis with respect to take diagonal. Default: 0. + axis2 (int, optional): The second axis with respect to take diagonal. Default: 1. + name (str, optional): Name for the operation (optional, default is None). For more information, please refer to :ref:`api_guide_Name`. Returns: Tensor: the output data type is the same as input data type. @@ -2785,11 +2762,11 @@ def diagonal(x, offset=0, axis1=0, axis2=1, name=None): - If offset < 0, it is below the main diagonal. Args: - x(Tensor): The input tensor x. Must be at least 2-dimensional. The input data type should be bool, int32, int64, float16, float32, float64. - offset(int, optional): Which diagonals in input tensor x will be taken. Default: 0 (main diagonals). - axis1(int, optional): The first axis with respect to take diagonal. Default: 0. - axis2(int, optional): The second axis with respect to take diagonal. Default: 1. - name (str, optional): Normally there is no need for user to set this property. For more information, please refer to :ref:`api_guide_Name`. Default: None. + x (Tensor): The input tensor x. Must be at least 2-dimensional. The input data type should be bool, int32, int64, float16, float32, float64. + offset (int, optional): Which diagonals in input tensor x will be taken. Default: 0 (main diagonals). + axis1 (int, optional): The first axis with respect to take diagonal. Default: 0. + axis2 (int, optional): The second axis with respect to take diagonal. Default: 1. + name (str, optional): Name for the operation (optional, default is None). For more information, please refer to :ref:`api_guide_Name`. Returns: Tensor: a partial view of input tensor in specify two dimensions, the output data type is the same as input data type. @@ -2893,9 +2870,7 @@ def kron(x, y, name=None): y (Tensor): the second operand of kron op, data type: float16, float32, float64, int32 or int64. Its data type should be the same with x. - name(str, optional): The default value is None. Normally there is no - need for user to set this property. For more information, please - refer to :ref:`api_guide_Name`. + name (str, optional): Name for the operation (optional, default is None). For more information, please refer to :ref:`api_guide_Name`. Returns: Tensor: The output of kron op, data type: float16, float32, float64, int32 or int64. Its data is the same with x. @@ -3155,19 +3130,18 @@ def prod(x, axis=None, keepdim=False, dtype=None, name=None): Compute the product of tensor elements over the given axis. Args: - x(Tensor): The input tensor, its data type should be float32, float64, int32, int64. - axis(int|list|tuple, optional): The axis along which the product is computed. If :attr:`None`, + x (Tensor): The input tensor, its data type should be float32, float64, int32, int64. + axis (int|list|tuple, optional): The axis along which the product is computed. If :attr:`None`, multiply all elements of `x` and return a Tensor with a single element, otherwise must be in the range :math:`[-x.ndim, x.ndim)`. If :math:`axis[i]<0`, the axis to reduce is :math:`x.ndim + axis[i]`. Default is None. - dtype(str|np.dtype, optional): The desired date type of returned tensor, can be float32, float64, + dtype (str|np.dtype, optional): The desired date type of returned tensor, can be float32, float64, int32, int64. If specified, the input tensor is casted to dtype before operator performed. This is very useful for avoiding data type overflows. The default value is None, the dtype of output is the same as input Tensor `x`. - keepdim(bool, optional): Whether to reserve the reduced dimension in the output Tensor. The result + keepdim (bool, optional): Whether to reserve the reduced dimension in the output Tensor. The result tensor will have one fewer dimension than the input unless `keepdim` is true. Default is False. - name(string, optional): The default value is None. Normally there is no need for user to set this property. - For more information, please refer to :ref:`api_guide_Name` . + name (str, optional): Name for the operation (optional, default is None). For more information, please refer to :ref:`api_guide_Name`. Returns: Tensor, result of product on the specified dim of input tensor. @@ -3253,9 +3227,8 @@ def sign(x, name=None): This OP returns sign of every element in `x`: 1 for positive, -1 for negative and 0 for zero. Args: - x(Tensor): The input tensor. The data type can be float16, float32 or float64. - name (str, optional): The default value is None. Normally there is no need for user to - set this property. For more information, please refer to :ref:`api_guide_Name` + x (Tensor): The input tensor. The data type can be float16, float32 or float64. + name (str, optional): Name for the operation (optional, default is None). For more information, please refer to :ref:`api_guide_Name`. Returns: Tensor: The output sign tensor with identical shape and data type to the input :attr:`x`. @@ -3338,7 +3311,7 @@ def increment(x, value=1.0, name=None): Args: x (Tensor): A tensor that must always contain only one element, its data type supports float32, float64, int32 and int64. - value(float, optional): The amount to increment the data of :attr:`x`. Default: 1.0. + value (float, optional): The amount to increment the data of :attr:`x`. Default: 1.0. name (str, optional): Name for the operation (optional, default is None). For more information, please refer to :ref:`api_guide_Name`. Returns: @@ -3386,8 +3359,7 @@ def all(x, axis=None, keepdim=False, name=None): output Tensor. The result Tensor will have one fewer dimension than the :attr:`x` unless :attr:`keepdim` is true, default value is False. - name (str, optional): The default value is None. Normally there is no need for - user to set this property. For more information, please refer to :ref:`api_guide_Name` + name (str, optional): Name for the operation (optional, default is None). For more information, please refer to :ref:`api_guide_Name`. Returns: Tensor: Results the ``logical and`` on the specified axis of input Tensor `x`, it's data type is bool. @@ -3483,8 +3455,7 @@ def any(x, axis=None, keepdim=False, name=None): output Tensor. The result Tensor will have one fewer dimension than the :attr:`x` unless :attr:`keepdim` is true, default value is False. - name (str, optional): The default value is None. Normally there is no need for - user to set this property. For more information, please refer to :ref:`api_guide_Name` + name (str, optional): Name for the operation (optional, default is None). For more information, please refer to :ref:`api_guide_Name`. Returns: Tensor: Results the ``logical or`` on the specified axis of input Tensor `x`, it's data type is bool. @@ -3599,8 +3570,7 @@ def conj(x, name=None): Args: x (Tensor): The input tensor which hold the complex numbers. Optional data types are: complex64, complex128, float32, float64, int32 or int64. - name (str, optional): The default value is None. Normally there is no need for - user to set this property. For more information, please refer to :ref:`api_guide_Name` + name (str, optional): Name for the operation (optional, default is None). For more information, please refer to :ref:`api_guide_Name`. Returns: out (Tensor): The conjugate of input. The shape and data type is the same with input. @@ -3645,8 +3615,7 @@ def digamma(x, name=None): Args: x (Tensor): Input Tensor. Must be one of the following types: float32, float64. - name(str, optional): The default value is None. Normally there is no need for - user to set this property. For more information, please refer to :ref:`api_guide_Name` + name (str, optional): Name for the operation (optional, default is None). For more information, please refer to :ref:`api_guide_Name`. Returns: Tensor, the digamma of the input Tensor, the shape and data type is the same with input. @@ -4201,18 +4170,17 @@ def diff(x, n=1, axis=-1, prepend=None, append=None, name=None): Only n=1 is currently supported. Args: - x(Tensor): The input tensor to compute the forward difference on - n(int, optional): The number of times to recursively compute the difference. + x (Tensor): The input tensor to compute the forward difference on + n (int, optional): The number of times to recursively compute the difference. Only support n=1. Default:1 - axis(int, optional): The axis to compute the difference along. Default:-1 - prepend(Tensor, optional): The tensor to prepend to input along axis before computing the difference. + axis (int, optional): The axis to compute the difference along. Default:-1 + prepend (Tensor, optional): The tensor to prepend to input along axis before computing the difference. It's dimensions must be equivalent to that of x, and its shapes must match x's shape except on axis. - append(Tensor, optional): The tensor to append to input along axis before computing the difference, + append (Tensor, optional): The tensor to append to input along axis before computing the difference, It's dimensions must be equivalent to that of x, and its shapes must match x's shape except on axis. - name(str|None): A name for this layer(optional). If set None, - the layer will be named automatically. + name (str, optional): Name for the operation (optional, default is None). For more information, please refer to :ref:`api_guide_Name`. Returns: Tensor: The output tensor with same dtype with x. @@ -4418,7 +4386,7 @@ def frac(x, name=None): Args: x (Tensor): The input tensor, which data type should be int32, int64, float32, float64. - name: (str, optional): Name for operation (optional, default is None). For more + name (str, optional): Name for the operation (optional, default is None). For more information, please refer to :ref:`api_guide_Name`. Returns: Tensor: The output Tensor of frac. diff --git a/python/paddle/tests/test_model.py b/python/paddle/tests/test_model.py index ce3a3bd4b02fe..fd3cb83d24e8a 100644 --- a/python/paddle/tests/test_model.py +++ b/python/paddle/tests/test_model.py @@ -783,7 +783,7 @@ def test_export_deploy_model(self): feed={feed_target_names[0]: tensor_img}, fetch_list=fetch_targets) np.testing.assert_allclose( - results, ori_results, rtol=1e-5, atol=1e-7) + results, ori_results, rtol=1e-5, atol=1e-6) paddle.enable_static() diff --git a/python/paddle/utils/code_gen/api.yaml b/python/paddle/utils/code_gen/api.yaml index 58b80950e5529..d401e7c5190fe 100644 --- a/python/paddle/utils/code_gen/api.yaml +++ b/python/paddle/utils/code_gen/api.yaml @@ -382,6 +382,7 @@ func : ConvTransposeInferMeta kernel : func : conv2d_transpose + use_gpudnn : true backward : conv2d_transpose_grad - api : conv3d_transpose @@ -391,6 +392,7 @@ func : ConvTransposeInferMeta kernel : func : conv3d_transpose + use_gpudnn : true backward : conv3d_transpose_grad - api : copy_to @@ -1556,8 +1558,20 @@ func : PoolInferMeta kernel : func : pool2d + use_gpudnn : true backward : pool2d_grad +# Used in adaptive_avg_pool2d API +- api : pool2d_gpudnn_unused + args : (Tensor x, int[] kernel_size, int[] strides, int[] paddings, bool ceil_mode, bool exclusive, str data_format, str pooling_type, bool global_pooling, bool adaptive, str padding_algorithm) + output : Tensor(out) + infer_meta : + func : PoolInferMeta + kernel : + func : pool2d + use_gpudnn : false + backward : pool2d_grad_gpudnn_unused + - api : pool3d args : (Tensor x, int[] kernel_size, int[] strides, int[] paddings, bool ceil_mode, bool exclusive, str data_format, str pooling_type, bool global_pooling, bool adaptive, str padding_algorithm) output : Tensor(out) @@ -1565,6 +1579,7 @@ func : PoolInferMeta kernel : func : pool3d + use_gpudnn : true backward : pool3d_grad - api : pow @@ -1923,6 +1938,7 @@ func : SoftmaxInferMeta kernel : func : softmax + use_gpudnn : true backward : softmax_grad - api : split diff --git a/python/paddle/utils/code_gen/api_base.py b/python/paddle/utils/code_gen/api_base.py index 378ead7ff20aa..717870ee01d0a 100644 --- a/python/paddle/utils/code_gen/api_base.py +++ b/python/paddle/utils/code_gen/api_base.py @@ -238,7 +238,7 @@ def parse_kernel(self, kernel_config): 'backend': None, 'layout': None, 'data_type': None, - 'use_cudnn': 'false' + 'use_gpudnn': 'false' } if 'backend' in kernel_config and len(kernel_config['backend']) > 0: kernel['backend'] = kernel_config['backend'] @@ -248,10 +248,10 @@ def parse_kernel(self, kernel_config): kernel['data_type'] = kernel_config['data_type'] if 'param' in kernel_config: kernel['param'] = kernel_config['param'] - if 'use_cudnn' in kernel_config: - kernel['use_cudnn'] = kernel_config['use_cudnn'] - if isinstance(kernel['use_cudnn'], bool): - kernel['use_cudnn'] = str(kernel['use_cudnn']).lower() + if 'use_gpudnn' in kernel_config: + kernel['use_gpudnn'] = kernel_config['use_gpudnn'] + if isinstance(kernel['use_gpudnn'], bool): + kernel['use_gpudnn'] = str(kernel['use_gpudnn']).lower() kernel['func'] = [ kernel_fn.strip() for kernel_fn in kernel_config['func'].split(',') ] @@ -729,7 +729,7 @@ def gen_dense_tensor_kernel_code(self, code_indent, inplace_flag=False): self.outputs['types'], 'SetKernelOutput', code_indent, inplace_flag) api_func_name = self.get_api_func_name() + ('_' if inplace_flag else '') cudnn_args = '' if self.kernel[ - 'use_cudnn'] == 'false' else ', ' + self.kernel['use_cudnn'] + 'use_gpudnn'] == 'false' else ', ' + self.kernel['use_gpudnn'] return f""" {code_indent} VLOG(6) << "{self.api} API kernel key: [" << kernel_backend << ", " << kernel_layout << ", "<< kernel_data_type << "]"; {code_indent} const auto& kernel = phi::KernelFactory::Instance().SelectKernelOrThrowError( diff --git a/python/paddle/utils/code_gen/backward.yaml b/python/paddle/utils/code_gen/backward.yaml index dfdc2335ae180..3b47470139b90 100644 --- a/python/paddle/utils/code_gen/backward.yaml +++ b/python/paddle/utils/code_gen/backward.yaml @@ -272,7 +272,7 @@ param: [input, filter, grad_out] kernel : func : conv2d_grad_grad - use_cudnn : true + use_gpudnn : true optional : grad_input_grad, grad_filter_grad - backward_api : conv2d_transpose_grad @@ -283,6 +283,7 @@ func : ConvTransposeGradInferMeta kernel : func : conv2d_transpose_grad + use_gpudnn : true - backward_api : conv3d_transpose_grad forward : conv3d_transpose(Tensor x, Tensor filter, int[] strides, int[] paddings, int[] output_padding, int[] output_size, str padding_algorithm, int groups, int[] dilations, str data_format) -> Tensor(out) @@ -292,6 +293,7 @@ func : ConvTransposeGradInferMeta kernel : func : conv3d_transpose_grad + use_gpudnn : true - backward_api : cos_grad forward : cos (Tensor x) -> Tensor(out) @@ -1234,6 +1236,17 @@ func : PoolGradInferMeta kernel : func : pool2d_grad + use_gpudnn : true + +- backward_api : pool2d_grad_gpudnn_unused + forward : pool2d_gpudnn_unused(Tensor x, int[] kernel_size, int[] strides, int[] paddings, bool ceil_mode, bool exclusive, str data_format, str pooling_type, bool global_pooling, bool adaptive, str padding_algorithm) -> Tensor(out) + args : (Tensor x, Tensor out, Tensor out_grad, int[] kernel_size, int[] strides, int[] paddings, bool ceil_mode, bool exclusive, str data_format, str pooling_type, bool global_pooling, bool adaptive, str padding_algorithm) + output : Tensor(x_grad) + infer_meta : + func : PoolGradInferMeta + kernel : + func : pool2d_grad + use_gpudnn : false - backward_api : pool3d_grad forward : pool3d(Tensor x, int[] kernel_size, int[] strides, int[] paddings, bool ceil_mode, bool exclusive, str data_format, str pooling_type, bool global_pooling, bool adaptive, str padding_algorithm) -> Tensor(out) @@ -1243,6 +1256,7 @@ func : PoolGradInferMeta kernel : func : pool3d_grad + use_gpudnn : true - backward_api : pow_grad forward : pow(Tensor x, Scalar s) -> Tensor(out) @@ -1578,6 +1592,7 @@ param : [out] kernel : func : softmax_grad + use_gpudnn : true - backward_api : split_grad forward : split (Tensor x, IntArray num_or_sections, Scalar axis) -> Tensor[](out) diff --git a/python/paddle/utils/code_gen/sparse_api.yaml b/python/paddle/utils/code_gen/sparse_api.yaml index 100d7ad78319b..ca4330f2af362 100644 --- a/python/paddle/utils/code_gen/sparse_api.yaml +++ b/python/paddle/utils/code_gen/sparse_api.yaml @@ -65,3 +65,12 @@ args : (Tensor x) output : Tensor(out@SparseCsrTensor) invoke : to_sparse_csr_impl(x) + +- api: maxpool + args : (Tensor x, int[] kernel_sizes, int[] paddings, int[] dilations, int[] strides) + output : Tensor(out@SparseCooTensor), Tensor(rulebook@DenseTensor) + kernel : + func : sparse_maxpool + layout : x + intermediate : rulebook + backward : sparse_maxpool_grad diff --git a/python/paddle/utils/code_gen/sparse_bw_api.yaml b/python/paddle/utils/code_gen/sparse_bw_api.yaml index e3946cbf72bc2..74299ed3e39a0 100644 --- a/python/paddle/utils/code_gen/sparse_bw_api.yaml +++ b/python/paddle/utils/code_gen/sparse_bw_api.yaml @@ -32,6 +32,13 @@ output : Tensor(x_grad@DenseTensor) invoke : to_dense_impl(out_grad) +- backward_api : sparse_maxpool_grad + forward : sparse_maxpool(Tensor x, int[] kernel_sizes, int[] paddings, int[] dilations, int[] strides) -> Tensor(out@SparseCooTensor), Tensor(rulebook@DenseTensor) + args : (Tensor x, Tensor rulebook, Tensor out, Tensor out_grad, int[] kernel_sizes) + output : Tensor(x_grad@SparseCooTensor) + kernel : + func : sparse_maxpool_grad + - backward_api : sparse_relu_grad forward : sparse_relu(Tensor x) -> Tensor(out@SparseCooTensor) args : (Tensor x, Tensor out_grad) diff --git a/python/paddle/utils/code_gen/wrapped_infermeta_gen.py b/python/paddle/utils/code_gen/wrapped_infermeta_gen.py index b50db007d92e9..dd077552b7962 100644 --- a/python/paddle/utils/code_gen/wrapped_infermeta_gen.py +++ b/python/paddle/utils/code_gen/wrapped_infermeta_gen.py @@ -141,7 +141,8 @@ def generate_wrapped_infermeta_and_register(api_yaml_path, header_file_path, api_item) header_file.write(declare_code) source_file.write(defind_code) - infermeta_register_code = infermeta_register_code + register_code + if infermeta_register_code.find(register_code) == -1: + infermeta_register_code = infermeta_register_code + register_code header_file.write(namespace[1]) source_file.write(namespace[1]) diff --git a/python/paddle/vision/__init__.py b/python/paddle/vision/__init__.py index 3749e0f64fc6a..2f0052537e251 100644 --- a/python/paddle/vision/__init__.py +++ b/python/paddle/vision/__init__.py @@ -34,6 +34,12 @@ from .models import resnet50 # noqa: F401 from .models import resnet101 # noqa: F401 from .models import resnet152 # noqa: F401 +from .models import resnext50_32x4d # noqa: F401 +from .models import resnext50_64x4d # noqa: F401 +from .models import resnext101_32x4d # noqa: F401 +from .models import resnext101_64x4d # noqa: F401 +from .models import resnext152_32x4d # noqa: F401 +from .models import resnext152_64x4d # noqa: F401 from .models import wide_resnet50_2 # noqa: F401 from .models import wide_resnet101_2 # noqa: F401 from .models import MobileNetV1 # noqa: F401 @@ -61,13 +67,6 @@ from .models import densenet264 # noqa: F401 from .models import AlexNet # noqa: F401 from .models import alexnet # noqa: F401 -from .models import ResNeXt # noqa: F401 -from .models import resnext50_32x4d # noqa: F401 -from .models import resnext50_64x4d # noqa: F401 -from .models import resnext101_32x4d # noqa: F401 -from .models import resnext101_64x4d # noqa: F401 -from .models import resnext152_32x4d # noqa: F401 -from .models import resnext152_64x4d # noqa: F401 from .models import InceptionV3 # noqa: F401 from .models import inception_v3 # noqa: F401 from .models import GoogLeNet # noqa: F401 diff --git a/python/paddle/vision/models/__init__.py b/python/paddle/vision/models/__init__.py index 5ff3562e56ea8..85ff5f85dffd0 100644 --- a/python/paddle/vision/models/__init__.py +++ b/python/paddle/vision/models/__init__.py @@ -18,6 +18,12 @@ from .resnet import resnet50 # noqa: F401 from .resnet import resnet101 # noqa: F401 from .resnet import resnet152 # noqa: F401 +from .resnet import resnext50_32x4d # noqa: F401 +from .resnet import resnext50_64x4d # noqa: F401 +from .resnet import resnext101_32x4d # noqa: F401 +from .resnet import resnext101_64x4d # noqa: F401 +from .resnet import resnext152_32x4d # noqa: F401 +from .resnet import resnext152_64x4d # noqa: F401 from .resnet import wide_resnet50_2 # noqa: F401 from .resnet import wide_resnet101_2 # noqa: F401 from .mobilenetv1 import MobileNetV1 # noqa: F401 @@ -42,13 +48,6 @@ from .densenet import densenet264 # noqa: F401 from .alexnet import AlexNet # noqa: F401 from .alexnet import alexnet # noqa: F401 -from .resnext import ResNeXt # noqa: F401 -from .resnext import resnext50_32x4d # noqa: F401 -from .resnext import resnext50_64x4d # noqa: F401 -from .resnext import resnext101_32x4d # noqa: F401 -from .resnext import resnext101_64x4d # noqa: F401 -from .resnext import resnext152_32x4d # noqa: F401 -from .resnext import resnext152_64x4d # noqa: F401 from .inceptionv3 import InceptionV3 # noqa: F401 from .inceptionv3 import inception_v3 # noqa: F401 from .squeezenet import SqueezeNet # noqa: F401 @@ -72,6 +71,12 @@ 'resnet50', 'resnet101', 'resnet152', + 'resnext50_32x4d', + 'resnext50_64x4d', + 'resnext101_32x4d', + 'resnext101_64x4d', + 'resnext152_32x4d', + 'resnext152_64x4d', 'wide_resnet50_2', 'wide_resnet101_2', 'VGG', @@ -96,13 +101,6 @@ 'densenet264', 'AlexNet', 'alexnet', - 'ResNeXt', - 'resnext50_32x4d', - 'resnext50_64x4d', - 'resnext101_32x4d', - 'resnext101_64x4d', - 'resnext152_32x4d', - 'resnext152_64x4d', 'InceptionV3', 'inception_v3', 'SqueezeNet', diff --git a/python/paddle/vision/models/inceptionv3.py b/python/paddle/vision/models/inceptionv3.py index 9e8a8b814688c..27650dbe09f04 100644 --- a/python/paddle/vision/models/inceptionv3.py +++ b/python/paddle/vision/models/inceptionv3.py @@ -19,75 +19,60 @@ import math import paddle import paddle.nn as nn -from paddle.nn import Conv2D, BatchNorm, Linear, Dropout +from paddle.nn import Linear, Dropout from paddle.nn import AdaptiveAvgPool2D, MaxPool2D, AvgPool2D from paddle.nn.initializer import Uniform from paddle.fluid.param_attr import ParamAttr from paddle.utils.download import get_weights_path_from_url +from ..ops import ConvNormActivation __all__ = [] model_urls = { "inception_v3": - ("https://paddle-imagenet-models-name.bj.bcebos.com/dygraph/legendary_models/InceptionV3_pretrained.pdparams", - "e4d0905a818f6bb7946e881777a8a935") + ("https://paddle-hapi.bj.bcebos.com/models/inception_v3.pdparams", + "649a4547c3243e8b59c656f41fe330b8") } -class ConvBNLayer(nn.Layer): - def __init__(self, - num_channels, - num_filters, - filter_size, - stride=1, - padding=0, - groups=1, - act="relu"): - super().__init__() - self.act = act - self.conv = Conv2D( - in_channels=num_channels, - out_channels=num_filters, - kernel_size=filter_size, - stride=stride, - padding=padding, - groups=groups, - bias_attr=False) - self.bn = BatchNorm(num_filters) - self.relu = nn.ReLU() - - def forward(self, x): - x = self.conv(x) - x = self.bn(x) - if self.act: - x = self.relu(x) - return x - - class InceptionStem(nn.Layer): def __init__(self): super().__init__() - self.conv_1a_3x3 = ConvBNLayer( - num_channels=3, num_filters=32, filter_size=3, stride=2, act="relu") - self.conv_2a_3x3 = ConvBNLayer( - num_channels=32, - num_filters=32, - filter_size=3, + self.conv_1a_3x3 = ConvNormActivation( + in_channels=3, + out_channels=32, + kernel_size=3, + stride=2, + padding=0, + activation_layer=nn.ReLU) + self.conv_2a_3x3 = ConvNormActivation( + in_channels=32, + out_channels=32, + kernel_size=3, stride=1, - act="relu") - self.conv_2b_3x3 = ConvBNLayer( - num_channels=32, - num_filters=64, - filter_size=3, + padding=0, + activation_layer=nn.ReLU) + self.conv_2b_3x3 = ConvNormActivation( + in_channels=32, + out_channels=64, + kernel_size=3, padding=1, - act="relu") + activation_layer=nn.ReLU) self.max_pool = MaxPool2D(kernel_size=3, stride=2, padding=0) - self.conv_3b_1x1 = ConvBNLayer( - num_channels=64, num_filters=80, filter_size=1, act="relu") - self.conv_4a_3x3 = ConvBNLayer( - num_channels=80, num_filters=192, filter_size=3, act="relu") + self.conv_3b_1x1 = ConvNormActivation( + in_channels=64, + out_channels=80, + kernel_size=1, + padding=0, + activation_layer=nn.ReLU) + self.conv_4a_3x3 = ConvNormActivation( + in_channels=80, + out_channels=192, + kernel_size=3, + padding=0, + activation_layer=nn.ReLU) def forward(self, x): x = self.conv_1a_3x3(x) @@ -103,47 +88,53 @@ def forward(self, x): class InceptionA(nn.Layer): def __init__(self, num_channels, pool_features): super().__init__() - self.branch1x1 = ConvBNLayer( - num_channels=num_channels, - num_filters=64, - filter_size=1, - act="relu") - self.branch5x5_1 = ConvBNLayer( - num_channels=num_channels, - num_filters=48, - filter_size=1, - act="relu") - self.branch5x5_2 = ConvBNLayer( - num_channels=48, - num_filters=64, - filter_size=5, + self.branch1x1 = ConvNormActivation( + in_channels=num_channels, + out_channels=64, + kernel_size=1, + padding=0, + activation_layer=nn.ReLU) + + self.branch5x5_1 = ConvNormActivation( + in_channels=num_channels, + out_channels=48, + kernel_size=1, + padding=0, + activation_layer=nn.ReLU) + self.branch5x5_2 = ConvNormActivation( + in_channels=48, + out_channels=64, + kernel_size=5, padding=2, - act="relu") - - self.branch3x3dbl_1 = ConvBNLayer( - num_channels=num_channels, - num_filters=64, - filter_size=1, - act="relu") - self.branch3x3dbl_2 = ConvBNLayer( - num_channels=64, - num_filters=96, - filter_size=3, + activation_layer=nn.ReLU) + + self.branch3x3dbl_1 = ConvNormActivation( + in_channels=num_channels, + out_channels=64, + kernel_size=1, + padding=0, + activation_layer=nn.ReLU) + self.branch3x3dbl_2 = ConvNormActivation( + in_channels=64, + out_channels=96, + kernel_size=3, padding=1, - act="relu") - self.branch3x3dbl_3 = ConvBNLayer( - num_channels=96, - num_filters=96, - filter_size=3, + activation_layer=nn.ReLU) + self.branch3x3dbl_3 = ConvNormActivation( + in_channels=96, + out_channels=96, + kernel_size=3, padding=1, - act="relu") + activation_layer=nn.ReLU) + self.branch_pool = AvgPool2D( kernel_size=3, stride=1, padding=1, exclusive=False) - self.branch_pool_conv = ConvBNLayer( - num_channels=num_channels, - num_filters=pool_features, - filter_size=1, - act="relu") + self.branch_pool_conv = ConvNormActivation( + in_channels=num_channels, + out_channels=pool_features, + kernel_size=1, + padding=0, + activation_layer=nn.ReLU) def forward(self, x): branch1x1 = self.branch1x1(x) @@ -164,29 +155,34 @@ def forward(self, x): class InceptionB(nn.Layer): def __init__(self, num_channels): super().__init__() - self.branch3x3 = ConvBNLayer( - num_channels=num_channels, - num_filters=384, - filter_size=3, + self.branch3x3 = ConvNormActivation( + in_channels=num_channels, + out_channels=384, + kernel_size=3, stride=2, - act="relu") - self.branch3x3dbl_1 = ConvBNLayer( - num_channels=num_channels, - num_filters=64, - filter_size=1, - act="relu") - self.branch3x3dbl_2 = ConvBNLayer( - num_channels=64, - num_filters=96, - filter_size=3, + padding=0, + activation_layer=nn.ReLU) + + self.branch3x3dbl_1 = ConvNormActivation( + in_channels=num_channels, + out_channels=64, + kernel_size=1, + padding=0, + activation_layer=nn.ReLU) + self.branch3x3dbl_2 = ConvNormActivation( + in_channels=64, + out_channels=96, + kernel_size=3, padding=1, - act="relu") - self.branch3x3dbl_3 = ConvBNLayer( - num_channels=96, - num_filters=96, - filter_size=3, + activation_layer=nn.ReLU) + self.branch3x3dbl_3 = ConvNormActivation( + in_channels=96, + out_channels=96, + kernel_size=3, stride=2, - act="relu") + padding=0, + activation_layer=nn.ReLU) + self.branch_pool = MaxPool2D(kernel_size=3, stride=2) def forward(self, x): @@ -206,70 +202,74 @@ def forward(self, x): class InceptionC(nn.Layer): def __init__(self, num_channels, channels_7x7): super().__init__() - self.branch1x1 = ConvBNLayer( - num_channels=num_channels, - num_filters=192, - filter_size=1, - act="relu") - - self.branch7x7_1 = ConvBNLayer( - num_channels=num_channels, - num_filters=channels_7x7, - filter_size=1, + self.branch1x1 = ConvNormActivation( + in_channels=num_channels, + out_channels=192, + kernel_size=1, + padding=0, + activation_layer=nn.ReLU) + + self.branch7x7_1 = ConvNormActivation( + in_channels=num_channels, + out_channels=channels_7x7, + kernel_size=1, stride=1, - act="relu") - self.branch7x7_2 = ConvBNLayer( - num_channels=channels_7x7, - num_filters=channels_7x7, - filter_size=(1, 7), + padding=0, + activation_layer=nn.ReLU) + self.branch7x7_2 = ConvNormActivation( + in_channels=channels_7x7, + out_channels=channels_7x7, + kernel_size=(1, 7), stride=1, padding=(0, 3), - act="relu") - self.branch7x7_3 = ConvBNLayer( - num_channels=channels_7x7, - num_filters=192, - filter_size=(7, 1), + activation_layer=nn.ReLU) + self.branch7x7_3 = ConvNormActivation( + in_channels=channels_7x7, + out_channels=192, + kernel_size=(7, 1), stride=1, padding=(3, 0), - act="relu") - - self.branch7x7dbl_1 = ConvBNLayer( - num_channels=num_channels, - num_filters=channels_7x7, - filter_size=1, - act="relu") - self.branch7x7dbl_2 = ConvBNLayer( - num_channels=channels_7x7, - num_filters=channels_7x7, - filter_size=(7, 1), + activation_layer=nn.ReLU) + + self.branch7x7dbl_1 = ConvNormActivation( + in_channels=num_channels, + out_channels=channels_7x7, + kernel_size=1, + padding=0, + activation_layer=nn.ReLU) + self.branch7x7dbl_2 = ConvNormActivation( + in_channels=channels_7x7, + out_channels=channels_7x7, + kernel_size=(7, 1), padding=(3, 0), - act="relu") - self.branch7x7dbl_3 = ConvBNLayer( - num_channels=channels_7x7, - num_filters=channels_7x7, - filter_size=(1, 7), + activation_layer=nn.ReLU) + self.branch7x7dbl_3 = ConvNormActivation( + in_channels=channels_7x7, + out_channels=channels_7x7, + kernel_size=(1, 7), padding=(0, 3), - act="relu") - self.branch7x7dbl_4 = ConvBNLayer( - num_channels=channels_7x7, - num_filters=channels_7x7, - filter_size=(7, 1), + activation_layer=nn.ReLU) + self.branch7x7dbl_4 = ConvNormActivation( + in_channels=channels_7x7, + out_channels=channels_7x7, + kernel_size=(7, 1), padding=(3, 0), - act="relu") - self.branch7x7dbl_5 = ConvBNLayer( - num_channels=channels_7x7, - num_filters=192, - filter_size=(1, 7), + activation_layer=nn.ReLU) + self.branch7x7dbl_5 = ConvNormActivation( + in_channels=channels_7x7, + out_channels=192, + kernel_size=(1, 7), padding=(0, 3), - act="relu") + activation_layer=nn.ReLU) self.branch_pool = AvgPool2D( kernel_size=3, stride=1, padding=1, exclusive=False) - self.branch_pool_conv = ConvBNLayer( - num_channels=num_channels, - num_filters=192, - filter_size=1, - act="relu") + self.branch_pool_conv = ConvNormActivation( + in_channels=num_channels, + out_channels=192, + kernel_size=1, + padding=0, + activation_layer=nn.ReLU) def forward(self, x): branch1x1 = self.branch1x1(x) @@ -296,40 +296,46 @@ def forward(self, x): class InceptionD(nn.Layer): def __init__(self, num_channels): super().__init__() - self.branch3x3_1 = ConvBNLayer( - num_channels=num_channels, - num_filters=192, - filter_size=1, - act="relu") - self.branch3x3_2 = ConvBNLayer( - num_channels=192, - num_filters=320, - filter_size=3, + self.branch3x3_1 = ConvNormActivation( + in_channels=num_channels, + out_channels=192, + kernel_size=1, + padding=0, + activation_layer=nn.ReLU) + self.branch3x3_2 = ConvNormActivation( + in_channels=192, + out_channels=320, + kernel_size=3, stride=2, - act="relu") - self.branch7x7x3_1 = ConvBNLayer( - num_channels=num_channels, - num_filters=192, - filter_size=1, - act="relu") - self.branch7x7x3_2 = ConvBNLayer( - num_channels=192, - num_filters=192, - filter_size=(1, 7), + padding=0, + activation_layer=nn.ReLU) + + self.branch7x7x3_1 = ConvNormActivation( + in_channels=num_channels, + out_channels=192, + kernel_size=1, + padding=0, + activation_layer=nn.ReLU) + self.branch7x7x3_2 = ConvNormActivation( + in_channels=192, + out_channels=192, + kernel_size=(1, 7), padding=(0, 3), - act="relu") - self.branch7x7x3_3 = ConvBNLayer( - num_channels=192, - num_filters=192, - filter_size=(7, 1), + activation_layer=nn.ReLU) + self.branch7x7x3_3 = ConvNormActivation( + in_channels=192, + out_channels=192, + kernel_size=(7, 1), padding=(3, 0), - act="relu") - self.branch7x7x3_4 = ConvBNLayer( - num_channels=192, - num_filters=192, - filter_size=3, + activation_layer=nn.ReLU) + self.branch7x7x3_4 = ConvNormActivation( + in_channels=192, + out_channels=192, + kernel_size=3, stride=2, - act="relu") + padding=0, + activation_layer=nn.ReLU) + self.branch_pool = MaxPool2D(kernel_size=3, stride=2) def forward(self, x): @@ -350,59 +356,64 @@ def forward(self, x): class InceptionE(nn.Layer): def __init__(self, num_channels): super().__init__() - self.branch1x1 = ConvBNLayer( - num_channels=num_channels, - num_filters=320, - filter_size=1, - act="relu") - self.branch3x3_1 = ConvBNLayer( - num_channels=num_channels, - num_filters=384, - filter_size=1, - act="relu") - self.branch3x3_2a = ConvBNLayer( - num_channels=384, - num_filters=384, - filter_size=(1, 3), + self.branch1x1 = ConvNormActivation( + in_channels=num_channels, + out_channels=320, + kernel_size=1, + padding=0, + activation_layer=nn.ReLU) + self.branch3x3_1 = ConvNormActivation( + in_channels=num_channels, + out_channels=384, + kernel_size=1, + padding=0, + activation_layer=nn.ReLU) + self.branch3x3_2a = ConvNormActivation( + in_channels=384, + out_channels=384, + kernel_size=(1, 3), padding=(0, 1), - act="relu") - self.branch3x3_2b = ConvBNLayer( - num_channels=384, - num_filters=384, - filter_size=(3, 1), + activation_layer=nn.ReLU) + self.branch3x3_2b = ConvNormActivation( + in_channels=384, + out_channels=384, + kernel_size=(3, 1), padding=(1, 0), - act="relu") - - self.branch3x3dbl_1 = ConvBNLayer( - num_channels=num_channels, - num_filters=448, - filter_size=1, - act="relu") - self.branch3x3dbl_2 = ConvBNLayer( - num_channels=448, - num_filters=384, - filter_size=3, + activation_layer=nn.ReLU) + + self.branch3x3dbl_1 = ConvNormActivation( + in_channels=num_channels, + out_channels=448, + kernel_size=1, + padding=0, + activation_layer=nn.ReLU) + self.branch3x3dbl_2 = ConvNormActivation( + in_channels=448, + out_channels=384, + kernel_size=3, padding=1, - act="relu") - self.branch3x3dbl_3a = ConvBNLayer( - num_channels=384, - num_filters=384, - filter_size=(1, 3), + activation_layer=nn.ReLU) + self.branch3x3dbl_3a = ConvNormActivation( + in_channels=384, + out_channels=384, + kernel_size=(1, 3), padding=(0, 1), - act="relu") - self.branch3x3dbl_3b = ConvBNLayer( - num_channels=384, - num_filters=384, - filter_size=(3, 1), + activation_layer=nn.ReLU) + self.branch3x3dbl_3b = ConvNormActivation( + in_channels=384, + out_channels=384, + kernel_size=(3, 1), padding=(1, 0), - act="relu") + activation_layer=nn.ReLU) + self.branch_pool = AvgPool2D( kernel_size=3, stride=1, padding=1, exclusive=False) - self.branch_pool_conv = ConvBNLayer( - num_channels=num_channels, - num_filters=192, - filter_size=1, - act="relu") + self.branch_pool_conv = ConvNormActivation( + in_channels=num_channels, + out_channels=192, + kernel_size=1, + padding=0, + activation_layer=nn.ReLU) def forward(self, x): branch1x1 = self.branch1x1(x) diff --git a/python/paddle/vision/models/mobilenetv1.py b/python/paddle/vision/models/mobilenetv1.py index 671a2cd8dfd5f..6d8d96952fab4 100644 --- a/python/paddle/vision/models/mobilenetv1.py +++ b/python/paddle/vision/models/mobilenetv1.py @@ -16,59 +16,31 @@ import paddle.nn as nn from paddle.utils.download import get_weights_path_from_url +from ..ops import ConvNormActivation __all__ = [] model_urls = { 'mobilenetv1_1.0': - ('https://paddle-hapi.bj.bcebos.com/models/mobilenet_v1_x1.0.pdparams', - '42a154c2f26f86e7457d6daded114e8c') + ('https://paddle-hapi.bj.bcebos.com/models/mobilenetv1_1.0.pdparams', + '3033ab1975b1670bef51545feb65fc45') } -class ConvBNLayer(nn.Layer): - def __init__(self, - in_channels, - out_channels, - kernel_size, - stride, - padding, - num_groups=1): - super(ConvBNLayer, self).__init__() - - self._conv = nn.Conv2D( - in_channels, - out_channels, - kernel_size, - stride=stride, - padding=padding, - groups=num_groups, - bias_attr=False) - - self._norm_layer = nn.BatchNorm2D(out_channels) - self._act = nn.ReLU() - - def forward(self, x): - x = self._conv(x) - x = self._norm_layer(x) - x = self._act(x) - return x - - class DepthwiseSeparable(nn.Layer): def __init__(self, in_channels, out_channels1, out_channels2, num_groups, stride, scale): super(DepthwiseSeparable, self).__init__() - self._depthwise_conv = ConvBNLayer( + self._depthwise_conv = ConvNormActivation( in_channels, int(out_channels1 * scale), kernel_size=3, stride=stride, padding=1, - num_groups=int(num_groups * scale)) + groups=int(num_groups * scale)) - self._pointwise_conv = ConvBNLayer( + self._pointwise_conv = ConvNormActivation( int(out_channels1 * scale), int(out_channels2 * scale), kernel_size=1, @@ -94,9 +66,15 @@ class MobileNetV1(nn.Layer): Examples: .. code-block:: python + import paddle from paddle.vision.models import MobileNetV1 model = MobileNetV1() + + x = paddle.rand([1, 3, 224, 224]) + out = model(x) + + print(out.shape) """ def __init__(self, scale=1.0, num_classes=1000, with_pool=True): @@ -106,7 +84,7 @@ def __init__(self, scale=1.0, num_classes=1000, with_pool=True): self.num_classes = num_classes self.with_pool = with_pool - self.conv1 = ConvBNLayer( + self.conv1 = ConvNormActivation( in_channels=3, out_channels=int(32 * scale), kernel_size=3, @@ -257,6 +235,7 @@ def mobilenet_v1(pretrained=False, scale=1.0, **kwargs): Examples: .. code-block:: python + import paddle from paddle.vision.models import mobilenet_v1 # build model @@ -266,7 +245,12 @@ def mobilenet_v1(pretrained=False, scale=1.0, **kwargs): # model = mobilenet_v1(pretrained=True) # build mobilenet v1 with scale=0.5 - model = mobilenet_v1(scale=0.5) + model_scale = mobilenet_v1(scale=0.5) + + x = paddle.rand([1, 3, 224, 224]) + out = model(x) + + print(out.shape) """ model = _mobilenet( 'mobilenetv1_' + str(scale), pretrained, scale=scale, **kwargs) diff --git a/python/paddle/vision/models/mobilenetv2.py b/python/paddle/vision/models/mobilenetv2.py index 6c486037c7d30..9791462610deb 100644 --- a/python/paddle/vision/models/mobilenetv2.py +++ b/python/paddle/vision/models/mobilenetv2.py @@ -17,6 +17,7 @@ from paddle.utils.download import get_weights_path_from_url from .utils import _make_divisible +from ..ops import ConvNormActivation __all__ = [] @@ -27,29 +28,6 @@ } -class ConvBNReLU(nn.Sequential): - def __init__(self, - in_planes, - out_planes, - kernel_size=3, - stride=1, - groups=1, - norm_layer=nn.BatchNorm2D): - padding = (kernel_size - 1) // 2 - - super(ConvBNReLU, self).__init__( - nn.Conv2D( - in_planes, - out_planes, - kernel_size, - stride, - padding, - groups=groups, - bias_attr=False), - norm_layer(out_planes), - nn.ReLU6()) - - class InvertedResidual(nn.Layer): def __init__(self, inp, @@ -67,15 +45,20 @@ def __init__(self, layers = [] if expand_ratio != 1: layers.append( - ConvBNReLU( - inp, hidden_dim, kernel_size=1, norm_layer=norm_layer)) + ConvNormActivation( + inp, + hidden_dim, + kernel_size=1, + norm_layer=norm_layer, + activation_layer=nn.ReLU6)) layers.extend([ - ConvBNReLU( + ConvNormActivation( hidden_dim, hidden_dim, stride=stride, groups=hidden_dim, - norm_layer=norm_layer), + norm_layer=norm_layer, + activation_layer=nn.ReLU6), nn.Conv2D( hidden_dim, oup, 1, 1, 0, bias_attr=False), norm_layer(oup), @@ -90,23 +73,30 @@ def forward(self, x): class MobileNetV2(nn.Layer): - def __init__(self, scale=1.0, num_classes=1000, with_pool=True): - """MobileNetV2 model from - `"MobileNetV2: Inverted Residuals and Linear Bottlenecks" `_. + """MobileNetV2 model from + `"MobileNetV2: Inverted Residuals and Linear Bottlenecks" `_. + + Args: + scale (float): scale of channels in each layer. Default: 1.0. + num_classes (int): output dim of last fc layer. If num_classes <=0, last fc layer + will not be defined. Default: 1000. + with_pool (bool): use pool before the last fc layer or not. Default: True. - Args: - scale (float): scale of channels in each layer. Default: 1.0. - num_classes (int): output dim of last fc layer. If num_classes <=0, last fc layer - will not be defined. Default: 1000. - with_pool (bool): use pool before the last fc layer or not. Default: True. + Examples: + .. code-block:: python + + import paddle + from paddle.vision.models import MobileNetV2 - Examples: - .. code-block:: python + model = MobileNetV2() - from paddle.vision.models import MobileNetV2 + x = paddle.rand([1, 3, 224, 224]) + out = model(x) + + print(out.shape) + """ - model = MobileNetV2() - """ + def __init__(self, scale=1.0, num_classes=1000, with_pool=True): super(MobileNetV2, self).__init__() self.num_classes = num_classes self.with_pool = with_pool @@ -130,8 +120,12 @@ def __init__(self, scale=1.0, num_classes=1000, with_pool=True): self.last_channel = _make_divisible(last_channel * max(1.0, scale), round_nearest) features = [ - ConvBNReLU( - 3, input_channel, stride=2, norm_layer=norm_layer) + ConvNormActivation( + 3, + input_channel, + stride=2, + norm_layer=norm_layer, + activation_layer=nn.ReLU6) ] for t, c, n, s in inverted_residual_setting: @@ -148,11 +142,12 @@ def __init__(self, scale=1.0, num_classes=1000, with_pool=True): input_channel = output_channel features.append( - ConvBNReLU( + ConvNormActivation( input_channel, self.last_channel, kernel_size=1, - norm_layer=norm_layer)) + norm_layer=norm_layer, + activation_layer=nn.ReLU6)) self.features = nn.Sequential(*features) @@ -199,6 +194,7 @@ def mobilenet_v2(pretrained=False, scale=1.0, **kwargs): Examples: .. code-block:: python + import paddle from paddle.vision.models import mobilenet_v2 # build model @@ -209,6 +205,11 @@ def mobilenet_v2(pretrained=False, scale=1.0, **kwargs): # build mobilenet v2 with scale=0.5 model = mobilenet_v2(scale=0.5) + + x = paddle.rand([1, 3, 224, 224]) + out = model(x) + + print(out.shape) """ model = _mobilenet( 'mobilenetv2_' + str(scale), pretrained, scale=scale, **kwargs) diff --git a/python/paddle/vision/models/resnet.py b/python/paddle/vision/models/resnet.py index 5921ae10eedef..27536b6a9c64f 100644 --- a/python/paddle/vision/models/resnet.py +++ b/python/paddle/vision/models/resnet.py @@ -33,12 +33,30 @@ '02f35f034ca3858e1e54d4036443c92d'), 'resnet152': ('https://paddle-hapi.bj.bcebos.com/models/resnet152.pdparams', '7ad16a2f1e7333859ff986138630fd7a'), - 'wide_resnet50_2': - ('https://paddle-hapi.bj.bcebos.com/models/wide_resnet50_2.pdparams', - '0282f804d73debdab289bd9fea3fa6dc'), - 'wide_resnet101_2': - ('https://paddle-hapi.bj.bcebos.com/models/wide_resnet101_2.pdparams', - 'd4360a2d23657f059216f5d5a1a9ac93'), + 'resnext50_32x4d': + ('https://paddle-hapi.bj.bcebos.com/models/resnext50_32x4d.pdparams', + 'dc47483169be7d6f018fcbb7baf8775d'), + "resnext50_64x4d": + ('https://paddle-hapi.bj.bcebos.com/models/resnext50_64x4d.pdparams', + '063d4b483e12b06388529450ad7576db'), + 'resnext101_32x4d': ( + 'https://paddle-hapi.bj.bcebos.com/models/resnext101_32x4d.pdparams', + '967b090039f9de2c8d06fe994fb9095f'), + 'resnext101_64x4d': ( + 'https://paddle-hapi.bj.bcebos.com/models/resnext101_64x4d.pdparams', + '98e04e7ca616a066699230d769d03008'), + 'resnext152_32x4d': ( + 'https://paddle-hapi.bj.bcebos.com/models/resnext152_32x4d.pdparams', + '18ff0beee21f2efc99c4b31786107121'), + 'resnext152_64x4d': ( + 'https://paddle-hapi.bj.bcebos.com/models/resnext152_64x4d.pdparams', + '77c4af00ca42c405fa7f841841959379'), + 'wide_resnet50_2': ( + 'https://paddle-hapi.bj.bcebos.com/models/wide_resnet50_2.pdparams', + '0282f804d73debdab289bd9fea3fa6dc'), + 'wide_resnet101_2': ( + 'https://paddle-hapi.bj.bcebos.com/models/wide_resnet101_2.pdparams', + 'd4360a2d23657f059216f5d5a1a9ac93'), } @@ -158,11 +176,12 @@ class ResNet(nn.Layer): Args: Block (BasicBlock|BottleneckBlock): block module of model. - depth (int): layers of resnet, default: 50. - width (int): base width of resnet, default: 64. - num_classes (int): output dim of last fc layer. If num_classes <=0, last fc layer + depth (int, optional): layers of resnet, Default: 50. + width (int, optional): base width per convolution group for each convolution block, Default: 64. + num_classes (int, optional): output dim of last fc layer. If num_classes <=0, last fc layer will not be defined. Default: 1000. - with_pool (bool): use pool before the last fc layer or not. Default: True. + with_pool (bool, optional): use pool before the last fc layer or not. Default: True. + groups (int, optional): number of groups for each convolution block, Default: 1. Examples: .. code-block:: python @@ -171,16 +190,23 @@ class ResNet(nn.Layer): from paddle.vision.models import ResNet from paddle.vision.models.resnet import BottleneckBlock, BasicBlock + # build ResNet with 18 layers + resnet18 = ResNet(BasicBlock, 18) + + # build ResNet with 50 layers resnet50 = ResNet(BottleneckBlock, 50) + # build Wide ResNet model wide_resnet50_2 = ResNet(BottleneckBlock, 50, width=64*2) - resnet18 = ResNet(BasicBlock, 18) + # build ResNeXt model + resnext50_32x4d = ResNet(BottleneckBlock, 50, width=4, groups=32) x = paddle.rand([1, 3, 224, 224]) out = resnet18(x) print(out.shape) + # [1, 1000] """ @@ -189,7 +215,8 @@ def __init__(self, depth=50, width=64, num_classes=1000, - with_pool=True): + with_pool=True, + groups=1): super(ResNet, self).__init__() layer_cfg = { 18: [2, 2, 2, 2], @@ -199,7 +226,7 @@ def __init__(self, 152: [3, 8, 36, 3] } layers = layer_cfg[depth] - self.groups = 1 + self.groups = groups self.base_width = width self.num_classes = num_classes self.with_pool = with_pool @@ -300,7 +327,7 @@ def resnet18(pretrained=False, **kwargs): `"Deep Residual Learning for Image Recognition" `_ Args: - pretrained (bool): If True, returns a model pre-trained on ImageNet + pretrained (bool, optional): If True, returns a model pre-trained on ImageNet. Default: False. Examples: .. code-block:: python @@ -318,6 +345,7 @@ def resnet18(pretrained=False, **kwargs): out = model(x) print(out.shape) + # [1, 1000] """ return _resnet('resnet18', BasicBlock, 18, pretrained, **kwargs) @@ -327,7 +355,7 @@ def resnet34(pretrained=False, **kwargs): `"Deep Residual Learning for Image Recognition" `_ Args: - pretrained (bool): If True, returns a model pre-trained on ImageNet + pretrained (bool, optional): If True, returns a model pre-trained on ImageNet. Default: False. Examples: .. code-block:: python @@ -345,6 +373,7 @@ def resnet34(pretrained=False, **kwargs): out = model(x) print(out.shape) + # [1, 1000] """ return _resnet('resnet34', BasicBlock, 34, pretrained, **kwargs) @@ -354,7 +383,7 @@ def resnet50(pretrained=False, **kwargs): `"Deep Residual Learning for Image Recognition" `_ Args: - pretrained (bool): If True, returns a model pre-trained on ImageNet + pretrained (bool, optional): If True, returns a model pre-trained on ImageNet. Default: False. Examples: .. code-block:: python @@ -372,6 +401,7 @@ def resnet50(pretrained=False, **kwargs): out = model(x) print(out.shape) + # [1, 1000] """ return _resnet('resnet50', BottleneckBlock, 50, pretrained, **kwargs) @@ -381,7 +411,7 @@ def resnet101(pretrained=False, **kwargs): `"Deep Residual Learning for Image Recognition" `_ Args: - pretrained (bool): If True, returns a model pre-trained on ImageNet + pretrained (bool, optional): If True, returns a model pre-trained on ImageNet. Default: False. Examples: .. code-block:: python @@ -399,6 +429,7 @@ def resnet101(pretrained=False, **kwargs): out = model(x) print(out.shape) + # [1, 1000] """ return _resnet('resnet101', BottleneckBlock, 101, pretrained, **kwargs) @@ -408,7 +439,7 @@ def resnet152(pretrained=False, **kwargs): `"Deep Residual Learning for Image Recognition" `_ Args: - pretrained (bool): If True, returns a model pre-trained on ImageNet + pretrained (bool, optional): If True, returns a model pre-trained on ImageNet. Default: False. Examples: .. code-block:: python @@ -426,16 +457,201 @@ def resnet152(pretrained=False, **kwargs): out = model(x) print(out.shape) + # [1, 1000] """ return _resnet('resnet152', BottleneckBlock, 152, pretrained, **kwargs) +def resnext50_32x4d(pretrained=False, **kwargs): + """ResNeXt-50 32x4d model from + `"Aggregated Residual Transformations for Deep Neural Networks" `_ + + Args: + pretrained (bool, optional): If True, returns a model pre-trained on ImageNet. Default: False. + + Examples: + .. code-block:: python + + import paddle + from paddle.vision.models import resnext50_32x4d + + # build model + model = resnext50_32x4d() + + # build model and load imagenet pretrained weight + # model = resnext50_32x4d(pretrained=True) + + x = paddle.rand([1, 3, 224, 224]) + out = model(x) + + print(out.shape) + # [1, 1000] + """ + kwargs['groups'] = 32 + kwargs['width'] = 4 + return _resnet('resnext50_32x4d', BottleneckBlock, 50, pretrained, **kwargs) + + +def resnext50_64x4d(pretrained=False, **kwargs): + """ResNeXt-50 64x4d model from + `"Aggregated Residual Transformations for Deep Neural Networks" `_ + + Args: + pretrained (bool, optional): If True, returns a model pre-trained on ImageNet. Default: False. + + Examples: + .. code-block:: python + + import paddle + from paddle.vision.models import resnext50_64x4d + + # build model + model = resnext50_64x4d() + + # build model and load imagenet pretrained weight + # model = resnext50_64x4d(pretrained=True) + + x = paddle.rand([1, 3, 224, 224]) + out = model(x) + + print(out.shape) + # [1, 1000] + """ + kwargs['groups'] = 64 + kwargs['width'] = 4 + return _resnet('resnext50_64x4d', BottleneckBlock, 50, pretrained, **kwargs) + + +def resnext101_32x4d(pretrained=False, **kwargs): + """ResNeXt-101 32x4d model from + `"Aggregated Residual Transformations for Deep Neural Networks" `_ + + Args: + pretrained (bool, optional): If True, returns a model pre-trained on ImageNet. Default: False. + + Examples: + .. code-block:: python + + import paddle + from paddle.vision.models import resnext101_32x4d + + # build model + model = resnext101_32x4d() + + # build model and load imagenet pretrained weight + # model = resnext101_32x4d(pretrained=True) + + x = paddle.rand([1, 3, 224, 224]) + out = model(x) + + print(out.shape) + # [1, 1000] + """ + kwargs['groups'] = 32 + kwargs['width'] = 4 + return _resnet('resnext101_32x4d', BottleneckBlock, 101, pretrained, + **kwargs) + + +def resnext101_64x4d(pretrained=False, **kwargs): + """ResNeXt-101 64x4d model from + `"Aggregated Residual Transformations for Deep Neural Networks" `_ + + Args: + pretrained (bool, optional): If True, returns a model pre-trained on ImageNet. Default: False. + + Examples: + .. code-block:: python + + import paddle + from paddle.vision.models import resnext101_64x4d + + # build model + model = resnext101_64x4d() + + # build model and load imagenet pretrained weight + # model = resnext101_64x4d(pretrained=True) + + x = paddle.rand([1, 3, 224, 224]) + out = model(x) + + print(out.shape) + # [1, 1000] + """ + kwargs['groups'] = 64 + kwargs['width'] = 4 + return _resnet('resnext101_64x4d', BottleneckBlock, 101, pretrained, + **kwargs) + + +def resnext152_32x4d(pretrained=False, **kwargs): + """ResNeXt-152 32x4d model from + `"Aggregated Residual Transformations for Deep Neural Networks" `_ + + Args: + pretrained (bool, optional): If True, returns a model pre-trained on ImageNet. Default: False. + + Examples: + .. code-block:: python + + import paddle + from paddle.vision.models import resnext152_32x4d + + # build model + model = resnext152_32x4d() + + # build model and load imagenet pretrained weight + # model = resnext152_32x4d(pretrained=True) + + x = paddle.rand([1, 3, 224, 224]) + out = model(x) + + print(out.shape) + # [1, 1000] + """ + kwargs['groups'] = 32 + kwargs['width'] = 4 + return _resnet('resnext152_32x4d', BottleneckBlock, 152, pretrained, + **kwargs) + + +def resnext152_64x4d(pretrained=False, **kwargs): + """ResNeXt-152 64x4d model from + `"Aggregated Residual Transformations for Deep Neural Networks" `_ + + Args: + pretrained (bool, optional): If True, returns a model pre-trained on ImageNet. Default: False. + + Examples: + .. code-block:: python + + import paddle + from paddle.vision.models import resnext152_64x4d + + # build model + model = resnext152_64x4d() + + # build model and load imagenet pretrained weight + # model = resnext152_64x4d(pretrained=True) + + x = paddle.rand([1, 3, 224, 224]) + out = model(x) + + print(out.shape) + # [1, 1000] + """ + kwargs['groups'] = 64 + kwargs['width'] = 4 + return _resnet('resnext152_64x4d', BottleneckBlock, 152, pretrained, + **kwargs) + + def wide_resnet50_2(pretrained=False, **kwargs): """Wide ResNet-50-2 model from `"Wide Residual Networks" `_. Args: - pretrained (bool): If True, returns a model pre-trained on ImageNet + pretrained (bool, optional): If True, returns a model pre-trained on ImageNet. Default: False. Examples: .. code-block:: python @@ -453,6 +669,7 @@ def wide_resnet50_2(pretrained=False, **kwargs): out = model(x) print(out.shape) + # [1, 1000] """ kwargs['width'] = 64 * 2 return _resnet('wide_resnet50_2', BottleneckBlock, 50, pretrained, **kwargs) @@ -463,7 +680,7 @@ def wide_resnet101_2(pretrained=False, **kwargs): `"Wide Residual Networks" `_. Args: - pretrained (bool): If True, returns a model pre-trained on ImageNet + pretrained (bool, optional): If True, returns a model pre-trained on ImageNet. Default: False. Examples: .. code-block:: python @@ -481,6 +698,7 @@ def wide_resnet101_2(pretrained=False, **kwargs): out = model(x) print(out.shape) + # [1, 1000] """ kwargs['width'] = 64 * 2 return _resnet('wide_resnet101_2', BottleneckBlock, 101, pretrained, diff --git a/python/paddle/vision/models/resnext.py b/python/paddle/vision/models/resnext.py deleted file mode 100644 index 2e1073c8ac5ce..0000000000000 --- a/python/paddle/vision/models/resnext.py +++ /dev/null @@ -1,364 +0,0 @@ -# copyright (c) 2020 PaddlePaddle Authors. All Rights Reserve. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import math - -import paddle -import paddle.nn as nn -import paddle.nn.functional as F -from paddle.fluid.param_attr import ParamAttr -from paddle.nn import AdaptiveAvgPool2D, BatchNorm, Conv2D, Linear, MaxPool2D -from paddle.nn.initializer import Uniform -from paddle.utils.download import get_weights_path_from_url - -__all__ = [] - -model_urls = { - 'resnext50_32x4d': - ('https://paddle-imagenet-models-name.bj.bcebos.com/dygraph/ResNeXt50_32x4d_pretrained.pdparams', - 'bf04add2f7fd22efcbe91511bcd1eebe'), - "resnext50_64x4d": - ('https://paddle-imagenet-models-name.bj.bcebos.com/dygraph/ResNeXt50_64x4d_pretrained.pdparams', - '46307df0e2d6d41d3b1c1d22b00abc69'), - 'resnext101_32x4d': - ('https://paddle-imagenet-models-name.bj.bcebos.com/dygraph/ResNeXt101_32x4d_pretrained.pdparams', - '078ca145b3bea964ba0544303a43c36d'), - 'resnext101_64x4d': - ('https://paddle-imagenet-models-name.bj.bcebos.com/dygraph/ResNeXt101_64x4d_pretrained.pdparams', - '4edc0eb32d3cc5d80eff7cab32cd5c64'), - 'resnext152_32x4d': - ('https://paddle-imagenet-models-name.bj.bcebos.com/dygraph/ResNeXt152_32x4d_pretrained.pdparams', - '7971cc994d459af167c502366f866378'), - 'resnext152_64x4d': - ('https://paddle-imagenet-models-name.bj.bcebos.com/dygraph/ResNeXt152_64x4d_pretrained.pdparams', - '836943f03709efec364d486c57d132de'), -} - - -class ConvBNLayer(nn.Layer): - def __init__(self, - num_channels, - num_filters, - filter_size, - stride=1, - groups=1, - act=None): - super(ConvBNLayer, self).__init__() - self._conv = Conv2D( - in_channels=num_channels, - out_channels=num_filters, - kernel_size=filter_size, - stride=stride, - padding=(filter_size - 1) // 2, - groups=groups, - bias_attr=False) - self._batch_norm = BatchNorm(num_filters, act=act) - - def forward(self, inputs): - x = self._conv(inputs) - x = self._batch_norm(x) - return x - - -class BottleneckBlock(nn.Layer): - def __init__(self, - num_channels, - num_filters, - stride, - cardinality, - shortcut=True): - super(BottleneckBlock, self).__init__() - self.conv0 = ConvBNLayer( - num_channels=num_channels, - num_filters=num_filters, - filter_size=1, - act='relu') - self.conv1 = ConvBNLayer( - num_channels=num_filters, - num_filters=num_filters, - filter_size=3, - groups=cardinality, - stride=stride, - act='relu') - self.conv2 = ConvBNLayer( - num_channels=num_filters, - num_filters=num_filters * 2 if cardinality == 32 else num_filters, - filter_size=1, - act=None) - - if not shortcut: - self.short = ConvBNLayer( - num_channels=num_channels, - num_filters=num_filters * 2 - if cardinality == 32 else num_filters, - filter_size=1, - stride=stride) - - self.shortcut = shortcut - - def forward(self, inputs): - x = self.conv0(inputs) - conv1 = self.conv1(x) - conv2 = self.conv2(conv1) - - if self.shortcut: - short = inputs - else: - short = self.short(inputs) - - x = paddle.add(x=short, y=conv2) - x = F.relu(x) - return x - - -class ResNeXt(nn.Layer): - """ResNeXt model from - `"Aggregated Residual Transformations for Deep Neural Networks" `_ - - Args: - depth (int, optional): depth of resnext. Default: 50. - cardinality (int, optional): cardinality of resnext. Default: 32. - num_classes (int, optional): output dim of last fc layer. If num_classes <=0, last fc layer - will not be defined. Default: 1000. - with_pool (bool, optional): use pool before the last fc layer or not. Default: True. - - Examples: - .. code-block:: python - - import paddle - from paddle.vision.models import ResNeXt - - resnext50_32x4d = ResNeXt(depth=50, cardinality=32) - - """ - - def __init__(self, - depth=50, - cardinality=32, - num_classes=1000, - with_pool=True): - super(ResNeXt, self).__init__() - - self.depth = depth - self.cardinality = cardinality - self.num_classes = num_classes - self.with_pool = with_pool - - supported_depth = [50, 101, 152] - assert depth in supported_depth, \ - "supported layers are {} but input layer is {}".format( - supported_depth, depth) - supported_cardinality = [32, 64] - assert cardinality in supported_cardinality, \ - "supported cardinality is {} but input cardinality is {}" \ - .format(supported_cardinality, cardinality) - layer_cfg = {50: [3, 4, 6, 3], 101: [3, 4, 23, 3], 152: [3, 8, 36, 3]} - layers = layer_cfg[depth] - num_channels = [64, 256, 512, 1024] - num_filters = [128, 256, 512, - 1024] if cardinality == 32 else [256, 512, 1024, 2048] - - self.conv = ConvBNLayer( - num_channels=3, num_filters=64, filter_size=7, stride=2, act='relu') - self.pool2d_max = MaxPool2D(kernel_size=3, stride=2, padding=1) - - self.block_list = [] - for block in range(len(layers)): - shortcut = False - for i in range(layers[block]): - bottleneck_block = self.add_sublayer( - 'bb_%d_%d' % (block, i), - BottleneckBlock( - num_channels=num_channels[block] if i == 0 else - num_filters[block] * int(64 // self.cardinality), - num_filters=num_filters[block], - stride=2 if i == 0 and block != 0 else 1, - cardinality=self.cardinality, - shortcut=shortcut)) - self.block_list.append(bottleneck_block) - shortcut = True - - if with_pool: - self.pool2d_avg = AdaptiveAvgPool2D(1) - - if num_classes > 0: - self.pool2d_avg_channels = num_channels[-1] * 2 - stdv = 1.0 / math.sqrt(self.pool2d_avg_channels * 1.0) - self.out = Linear( - self.pool2d_avg_channels, - num_classes, - weight_attr=ParamAttr(initializer=Uniform(-stdv, stdv))) - - def forward(self, inputs): - with paddle.static.amp.fp16_guard(): - x = self.conv(inputs) - x = self.pool2d_max(x) - for block in self.block_list: - x = block(x) - if self.with_pool: - x = self.pool2d_avg(x) - if self.num_classes > 0: - x = paddle.reshape(x, shape=[-1, self.pool2d_avg_channels]) - x = self.out(x) - return x - - -def _resnext(arch, depth, cardinality, pretrained, **kwargs): - model = ResNeXt(depth=depth, cardinality=cardinality, **kwargs) - if pretrained: - assert arch in model_urls, "{} model do not have a pretrained model now, you should set pretrained=False".format( - arch) - weight_path = get_weights_path_from_url(model_urls[arch][0], - model_urls[arch][1]) - - param = paddle.load(weight_path) - model.set_dict(param) - - return model - - -def resnext50_32x4d(pretrained=False, **kwargs): - """ResNeXt-50 32x4d model from - `"Aggregated Residual Transformations for Deep Neural Networks" `_ - - Args: - pretrained (bool): If True, returns a model pre-trained on ImageNet - - Examples: - .. code-block:: python - - import paddle - from paddle.vision.models import resnext50_32x4d - - # build model - model = resnext50_32x4d() - - # build model and load imagenet pretrained weight - # model = resnext50_32x4d(pretrained=True) - """ - return _resnext('resnext50_32x4d', 50, 32, pretrained, **kwargs) - - -def resnext50_64x4d(pretrained=False, **kwargs): - """ResNeXt-50 64x4d model from - `"Aggregated Residual Transformations for Deep Neural Networks" `_ - - Args: - pretrained (bool): If True, returns a model pre-trained on ImageNet - - Examples: - .. code-block:: python - - import paddle - from paddle.vision.models import resnext50_64x4d - - # build model - model = resnext50_64x4d() - - # build model and load imagenet pretrained weight - # model = resnext50_64x4d(pretrained=True) - """ - return _resnext('resnext50_64x4d', 50, 64, pretrained, **kwargs) - - -def resnext101_32x4d(pretrained=False, **kwargs): - """ResNeXt-101 32x4d model from - `"Aggregated Residual Transformations for Deep Neural Networks" `_ - - Args: - pretrained (bool): If True, returns a model pre-trained on ImageNet - - Examples: - .. code-block:: python - - import paddle - from paddle.vision.models import resnext101_32x4d - - # build model - model = resnext101_32x4d() - - # build model and load imagenet pretrained weight - # model = resnext101_32x4d(pretrained=True) - """ - return _resnext('resnext101_32x4d', 101, 32, pretrained, **kwargs) - - -def resnext101_64x4d(pretrained=False, **kwargs): - """ResNeXt-101 64x4d model from - `"Aggregated Residual Transformations for Deep Neural Networks" `_ - - Args: - pretrained (bool): If True, returns a model pre-trained on ImageNet - - Examples: - .. code-block:: python - - import paddle - from paddle.vision.models import resnext101_64x4d - - # build model - model = resnext101_64x4d() - - # build model and load imagenet pretrained weight - # model = resnext101_64x4d(pretrained=True) - """ - return _resnext('resnext101_64x4d', 101, 64, pretrained, **kwargs) - - -def resnext152_32x4d(pretrained=False, **kwargs): - """ResNeXt-152 32x4d model from - `"Aggregated Residual Transformations for Deep Neural Networks" `_ - - Args: - pretrained (bool): If True, returns a model pre-trained on ImageNet - - Examples: - .. code-block:: python - - import paddle - from paddle.vision.models import resnext152_32x4d - - # build model - model = resnext152_32x4d() - - # build model and load imagenet pretrained weight - # model = resnext152_32x4d(pretrained=True) - """ - return _resnext('resnext152_32x4d', 152, 32, pretrained, **kwargs) - - -def resnext152_64x4d(pretrained=False, **kwargs): - """ResNeXt-152 64x4d model from - `"Aggregated Residual Transformations for Deep Neural Networks" `_ - - Args: - pretrained (bool): If True, returns a model pre-trained on ImageNet - - Examples: - .. code-block:: python - - import paddle - from paddle.vision.models import resnext152_64x4d - - # build model - model = resnext152_64x4d() - - # build model and load imagenet pretrained weight - # model = resnext152_64x4d(pretrained=True) - """ - return _resnext('resnext152_64x4d', 152, 64, pretrained, **kwargs) diff --git a/python/paddle/vision/models/shufflenetv2.py b/python/paddle/vision/models/shufflenetv2.py index 041f3fc749b6c..90e967ee22b35 100644 --- a/python/paddle/vision/models/shufflenetv2.py +++ b/python/paddle/vision/models/shufflenetv2.py @@ -18,37 +18,50 @@ import paddle import paddle.nn as nn -from paddle.fluid.param_attr import ParamAttr -from paddle.nn import AdaptiveAvgPool2D, BatchNorm, Conv2D, Linear, MaxPool2D +from paddle.nn import AdaptiveAvgPool2D, Linear, MaxPool2D from paddle.utils.download import get_weights_path_from_url +from ..ops import ConvNormActivation + __all__ = [] model_urls = { "shufflenet_v2_x0_25": ( - "https://paddle-imagenet-models-name.bj.bcebos.com/dygraph/ShuffleNetV2_x0_25_pretrained.pdparams", - "e753404cbd95027759c5f56ecd6c9c4b", ), + "https://paddle-hapi.bj.bcebos.com/models/shufflenet_v2_x0_25.pdparams", + "1e509b4c140eeb096bb16e214796d03b", ), "shufflenet_v2_x0_33": ( - "https://paddle-imagenet-models-name.bj.bcebos.com/dygraph/ShuffleNetV2_x0_33_pretrained.pdparams", - "776e3cf9a4923abdfce789c45b8fe1f2", ), + "https://paddle-hapi.bj.bcebos.com/models/shufflenet_v2_x0_33.pdparams", + "3d7b3ab0eaa5c0927ff1026d31b729bd", ), "shufflenet_v2_x0_5": ( - "https://paddle-imagenet-models-name.bj.bcebos.com/dygraph/ShuffleNetV2_x0_5_pretrained.pdparams", - "e3649cf531566917e2969487d2bc6b60", ), + "https://paddle-hapi.bj.bcebos.com/models/shufflenet_v2_x0_5.pdparams", + "5e5cee182a7793c4e4c73949b1a71bd4", ), "shufflenet_v2_x1_0": ( - "https://paddle-imagenet-models-name.bj.bcebos.com/dygraph/ShuffleNetV2_x1_0_pretrained.pdparams", - "7821c348ea34e58847c43a08a4ac0bdf", ), + "https://paddle-hapi.bj.bcebos.com/models/shufflenet_v2_x1_0.pdparams", + "122d42478b9e81eb49f8a9ede327b1a4", ), "shufflenet_v2_x1_5": ( - "https://paddle-imagenet-models-name.bj.bcebos.com/dygraph/ShuffleNetV2_x1_5_pretrained.pdparams", - "93a07fa557ab2d8803550f39e5b6c391", ), + "https://paddle-hapi.bj.bcebos.com/models/shufflenet_v2_x1_5.pdparams", + "faced5827380d73531d0ee027c67826d", ), "shufflenet_v2_x2_0": ( - "https://paddle-imagenet-models-name.bj.bcebos.com/dygraph/ShuffleNetV2_x2_0_pretrained.pdparams", - "4ab1f622fd0d341e0f84b4e057797563", ), + "https://paddle-hapi.bj.bcebos.com/models/shufflenet_v2_x2_0.pdparams", + "cd3dddcd8305e7bcd8ad14d1c69a5784", ), "shufflenet_v2_swish": ( - "https://paddle-imagenet-models-name.bj.bcebos.com/dygraph/ShuffleNetV2_swish_pretrained.pdparams", - "daff38b3df1b3748fccbb13cfdf02519", ), + "https://paddle-hapi.bj.bcebos.com/models/shufflenet_v2_swish.pdparams", + "adde0aa3b023e5b0c94a68be1c394b84", ), } +def create_activation_layer(act): + if act == "swish": + return nn.Swish + elif act == "relu": + return nn.ReLU + elif act is None: + return None + else: + raise RuntimeError( + "The activation function is not supported: {}".format(act)) + + def channel_shuffle(x, groups): batch_size, num_channels, height, width = x.shape[0:4] channels_per_group = num_channels // groups @@ -65,61 +78,37 @@ def channel_shuffle(x, groups): return x -class ConvBNLayer(nn.Layer): +class InvertedResidual(nn.Layer): def __init__(self, in_channels, out_channels, - kernel_size, stride, - padding, - groups=1, - act=None): - super(ConvBNLayer, self).__init__() - self._conv = Conv2D( - in_channels=in_channels, - out_channels=out_channels, - kernel_size=kernel_size, - stride=stride, - padding=padding, - groups=groups, - weight_attr=ParamAttr(initializer=nn.initializer.KaimingNormal()), - bias_attr=False, ) - - self._batch_norm = BatchNorm(out_channels, act=act) - - def forward(self, inputs): - x = self._conv(inputs) - x = self._batch_norm(x) - return x - - -class InvertedResidual(nn.Layer): - def __init__(self, in_channels, out_channels, stride, act="relu"): + activation_layer=nn.ReLU): super(InvertedResidual, self).__init__() - self._conv_pw = ConvBNLayer( + self._conv_pw = ConvNormActivation( in_channels=in_channels // 2, out_channels=out_channels // 2, kernel_size=1, stride=1, padding=0, groups=1, - act=act) - self._conv_dw = ConvBNLayer( + activation_layer=activation_layer) + self._conv_dw = ConvNormActivation( in_channels=out_channels // 2, out_channels=out_channels // 2, kernel_size=3, stride=stride, padding=1, groups=out_channels // 2, - act=None) - self._conv_linear = ConvBNLayer( + activation_layer=None) + self._conv_linear = ConvNormActivation( in_channels=out_channels // 2, out_channels=out_channels // 2, kernel_size=1, stride=1, padding=0, groups=1, - act=act) + activation_layer=activation_layer) def forward(self, inputs): x1, x2 = paddle.split( @@ -134,51 +123,55 @@ def forward(self, inputs): class InvertedResidualDS(nn.Layer): - def __init__(self, in_channels, out_channels, stride, act="relu"): + def __init__(self, + in_channels, + out_channels, + stride, + activation_layer=nn.ReLU): super(InvertedResidualDS, self).__init__() # branch1 - self._conv_dw_1 = ConvBNLayer( + self._conv_dw_1 = ConvNormActivation( in_channels=in_channels, out_channels=in_channels, kernel_size=3, stride=stride, padding=1, groups=in_channels, - act=None) - self._conv_linear_1 = ConvBNLayer( + activation_layer=None) + self._conv_linear_1 = ConvNormActivation( in_channels=in_channels, out_channels=out_channels // 2, kernel_size=1, stride=1, padding=0, groups=1, - act=act) + activation_layer=activation_layer) # branch2 - self._conv_pw_2 = ConvBNLayer( + self._conv_pw_2 = ConvNormActivation( in_channels=in_channels, out_channels=out_channels // 2, kernel_size=1, stride=1, padding=0, groups=1, - act=act) - self._conv_dw_2 = ConvBNLayer( + activation_layer=activation_layer) + self._conv_dw_2 = ConvNormActivation( in_channels=out_channels // 2, out_channels=out_channels // 2, kernel_size=3, stride=stride, padding=1, groups=out_channels // 2, - act=None) - self._conv_linear_2 = ConvBNLayer( + activation_layer=None) + self._conv_linear_2 = ConvNormActivation( in_channels=out_channels // 2, out_channels=out_channels // 2, kernel_size=1, stride=1, padding=0, groups=1, - act=act) + activation_layer=activation_layer) def forward(self, inputs): x1 = self._conv_dw_1(inputs) @@ -221,6 +214,7 @@ def __init__(self, scale=1.0, act="relu", num_classes=1000, with_pool=True): self.num_classes = num_classes self.with_pool = with_pool stage_repeats = [4, 8, 4] + activation_layer = create_activation_layer(act) if scale == 0.25: stage_out_channels = [-1, 24, 24, 48, 96, 512] @@ -238,13 +232,13 @@ def __init__(self, scale=1.0, act="relu", num_classes=1000, with_pool=True): raise NotImplementedError("This scale size:[" + str(scale) + "] is not implemented!") # 1. conv1 - self._conv1 = ConvBNLayer( + self._conv1 = ConvNormActivation( in_channels=3, out_channels=stage_out_channels[1], kernel_size=3, stride=2, padding=1, - act=act) + activation_layer=activation_layer) self._max_pool = MaxPool2D(kernel_size=3, stride=2, padding=1) # 2. bottleneck sequences @@ -257,7 +251,7 @@ def __init__(self, scale=1.0, act="relu", num_classes=1000, with_pool=True): in_channels=stage_out_channels[stage_id + 1], out_channels=stage_out_channels[stage_id + 2], stride=2, - act=act), + activation_layer=activation_layer), name=str(stage_id + 2) + "_" + str(i + 1)) else: block = self.add_sublayer( @@ -265,17 +259,17 @@ def __init__(self, scale=1.0, act="relu", num_classes=1000, with_pool=True): in_channels=stage_out_channels[stage_id + 2], out_channels=stage_out_channels[stage_id + 2], stride=1, - act=act), + activation_layer=activation_layer), name=str(stage_id + 2) + "_" + str(i + 1)) self._block_list.append(block) # 3. last_conv - self._last_conv = ConvBNLayer( + self._last_conv = ConvNormActivation( in_channels=stage_out_channels[-2], out_channels=stage_out_channels[-1], kernel_size=1, stride=1, padding=0, - act=act) + activation_layer=activation_layer) # 4. pool if with_pool: self._pool2d_avg = AdaptiveAvgPool2D(1) diff --git a/python/paddle/vision/ops.py b/python/paddle/vision/ops.py index 2d60fd4561480..e4dd4c797fef6 100644 --- a/python/paddle/vision/ops.py +++ b/python/paddle/vision/ops.py @@ -1335,13 +1335,13 @@ class ConvNormActivation(Sequential): Args: in_channels (int): Number of channels in the input image out_channels (int): Number of channels produced by the Convolution-Normalzation-Activation block - kernel_size: (int, optional): Size of the convolving kernel. Default: 3 - stride (int, optional): Stride of the convolution. Default: 1 - padding (int, tuple or str, optional): Padding added to all four sides of the input. Default: None, + kernel_size: (int|list|tuple, optional): Size of the convolving kernel. Default: 3 + stride (int|list|tuple, optional): Stride of the convolution. Default: 1 + padding (int|str|tuple|list, optional): Padding added to all four sides of the input. Default: None, in wich case it will calculated as ``padding = (kernel_size - 1) // 2 * dilation`` groups (int, optional): Number of blocked connections from input channels to output channels. Default: 1 norm_layer (Callable[..., paddle.nn.Layer], optional): Norm layer that will be stacked on top of the convolutiuon layer. - If ``None`` this layer wont be used. Default: ``paddle.nn.BatchNorm2d`` + If ``None`` this layer wont be used. Default: ``paddle.nn.BatchNorm2D`` activation_layer (Callable[..., paddle.nn.Layer], optional): Activation function which will be stacked on top of the normalization layer (if not ``None``), otherwise on top of the conv layer. If ``None`` this layer wont be used. Default: ``paddle.nn.ReLU`` dilation (int): Spacing between kernel elements. Default: 1 diff --git a/python/paddle/vision/transforms/functional_pil.py b/python/paddle/vision/transforms/functional_pil.py index b3ff37d7ea3bb..32f65fa1f846f 100644 --- a/python/paddle/vision/transforms/functional_pil.py +++ b/python/paddle/vision/transforms/functional_pil.py @@ -32,14 +32,25 @@ Sequence = collections.abc.Sequence Iterable = collections.abc.Iterable -_pil_interp_from_str = { - 'nearest': Image.NEAREST, - 'bilinear': Image.BILINEAR, - 'bicubic': Image.BICUBIC, - 'box': Image.BOX, - 'lanczos': Image.LANCZOS, - 'hamming': Image.HAMMING -} +try: + # PIL version >= "9.1.0" + _pil_interp_from_str = { + 'nearest': Image.Resampling.NEAREST, + 'bilinear': Image.Resampling.BILINEAR, + 'bicubic': Image.Resampling.BICUBIC, + 'box': Image.Resampling.BOX, + 'lanczos': Image.Resampling.LANCZOS, + 'hamming': Image.Resampling.HAMMING + } +except: + _pil_interp_from_str = { + 'nearest': Image.NEAREST, + 'bilinear': Image.BILINEAR, + 'bicubic': Image.BICUBIC, + 'box': Image.BOX, + 'lanczos': Image.LANCZOS, + 'hamming': Image.HAMMING + } __all__ = [] diff --git a/python/setup.py.in b/python/setup.py.in index e4637444be171..4cf8bc3fc6a2e 100755 --- a/python/setup.py.in +++ b/python/setup.py.in @@ -13,6 +13,7 @@ from contextlib import contextmanager from setuptools import Command from setuptools import setup, Distribution, Extension from setuptools.command.install import install as InstallCommandBase +from setuptools.command.egg_info import egg_info class BinaryDistribution(Distribution): @@ -311,6 +312,8 @@ packages=['paddle', 'paddle.distributed.auto_parallel.tuner', 'paddle.distributed.auto_parallel.cost', 'paddle.distributed.passes', + 'paddle.distributed.models', + 'paddle.distributed.models.moe', 'paddle.framework', 'paddle.jit', 'paddle.jit.dy2static', @@ -365,6 +368,10 @@ packages=['paddle', 'paddle.incubate.nn.functional', 'paddle.incubate.nn.layer', 'paddle.incubate.optimizer.functional', + 'paddle.incubate.distributed', + 'paddle.incubate.distributed.models', + 'paddle.incubate.distributed.models.moe', + 'paddle.incubate.distributed.models.moe.gate', 'paddle.io', 'paddle.optimizer', 'paddle.nn', @@ -678,6 +685,17 @@ class InstallHeaders(Command): def get_outputs(self): return self.outfiles +class EggInfo(egg_info): + """Copy license file into `.dist-info` folder.""" + + def run(self): + # don't duplicate license into `.dist-info` when building a distribution + if not self.distribution.have_run.get('install', True): + self.mkpath(self.egg_info) + self.copy_file("@PADDLE_SOURCE_DIR@/LICENSE", self.egg_info) + + egg_info.run(self) + # we redirect setuptools log for non-windows if sys.platform != 'win32': @contextmanager @@ -733,6 +751,7 @@ with redirect_stdout(): cmdclass={ 'install_headers': InstallHeaders, 'install': InstallCommand, + 'egg_info': EggInfo, }, entry_points={ 'console_scripts': [ diff --git a/tools/infrt/generate_phi_kernel_dialect.py b/tools/infrt/generate_phi_kernel_dialect.py index 0b67c6ba44a1d..b83bfe911aa48 100644 --- a/tools/infrt/generate_phi_kernel_dialect.py +++ b/tools/infrt/generate_phi_kernel_dialect.py @@ -20,12 +20,12 @@ #TODO @DannyIsFunny: more attr types need to be supported. attr_type_converter = { - "i": 'SI32Attr', - "b": 'BoolAttr', - "l": 'SI64Attr', - "f": 'F32Attr', - "NSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEE": 'StrAttr', - "St6vectorIiSaIiEE": 'I32ArrayAttr' + "int": 'SI32Attr', + "bool": 'BoolAttr', + "int64_t": 'SI64Attr', + "float": 'F32Attr', + "string": 'StrAttr', + "vector": 'I32ArrayAttr' } target_type_converter = {"CPU": "CPU", "GPU": "GPU", "Undefined": "UNK"} diff --git a/tools/static_mode_white_list.py b/tools/static_mode_white_list.py index 47b1ba5700e1b..aaa667595f94c 100755 --- a/tools/static_mode_white_list.py +++ b/tools/static_mode_white_list.py @@ -92,6 +92,7 @@ 'test_case', 'test_cast_op', 'test_center_loss', + 'test_channel_shuffle', 'test_cholesky_op', 'test_chunk_eval_op', 'test_chunk_op', @@ -386,6 +387,7 @@ 'test_partial_sum_op', 'test_pass_builder', 'test_pixel_shuffle', + 'test_pixel_unshuffle', 'test_polygon_box_transform', 'test_pool1d_api', 'test_pool2d_api',