Merge branch 'develop' of https://github.com/PaddlePaddle/Paddle into…

… add_complex_support_for_optest

CMakeLists.txt

@@ -92,6 +92,11 @@ message(STATUS "C compiler: ${CMAKE_C_COMPILER}, version: "
                "${CMAKE_C_COMPILER_ID} ${CMAKE_C_COMPILER_VERSION}")
 message(STATUS "AR tools: ${CMAKE_AR}")
 
+if((CMAKE_CXX_COMPILER_ID STREQUAL "GNU") AND CMAKE_CXX_COMPILER_VERSION
+                                              VERSION_GREATER 10.4)
+  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-error=uninitialized")
+endif()
+
 # MUSL build turn off warnings
 if(WITH_MUSL)
   set(CMAKE_CXX_FLAGS
@@ -246,6 +251,7 @@ option(WITH_DISTRIBUTE "Compile with distributed support" OFF)
 option(WITH_BRPC_RDMA "Use brpc rdma as the rpc protocal" OFF)
 option(ON_INFER "Turn on inference optimization and inference-lib generation"
        ON)
+option(WITH_CPP_DIST "Install PaddlePaddle C++ distribution" OFF)
 ################################ Internal Configurations #######################################
 option(WITH_NV_JETSON "Compile PaddlePaddle with NV JETSON" OFF)
 option(WITH_PROFILER "Compile PaddlePaddle with GPU profiler and gperftools"
@@ -662,6 +668,21 @@ if(WITH_STRIP)
   endif()
 endif()
 
+if(WITH_CPP_DIST)
+  # TODO(huangjiyi): Separate installing C++ distribution from python package
+  # installation and support for installing C++ distribution on more platforms.
+  if(NOT LINUX OR NOT WITH_PYTHON)
+    set(WITH_CPP_DIST
+        OFF
+        CACHE
+          STRING
+          "Currently C++ Distribution Generation is only available on Linux and compiling WITH_PYTHON=ON."
+          FORCE)
+  else()
+    include(paddle_lib)
+  endif()
+endif()
+
 add_subdirectory(paddle)
 if(WITH_PYTHON)
   add_subdirectory(python)

cmake/PaddleConfig.cmake.in

@@ -0,0 +1,33 @@
+# Paddle CMake configuration file
+# -------
+#
+# Finds the Paddle library
+#
+# This will define the following variables:
+#
+#   PADDLE_FOUND        -- True if the system has the Paddle library
+#   PADDLE_INCLUDE_DIRS -- The include directories for Paddle
+#   PADDLE_LIBRARIES    -- Libraries to link against
+
+get_filename_component(PADDLE_INSTALL_PREFIX "${CMAKE_CURRENT_LIST_FILE}/../.." ABSOLUTE)
+
+# include directories
+set(PADDLE_INCLUDE_DIRS 
+    ${PADDLE_INSTALL_PREFIX}/include
+    ${PADDLE_INSTALL_PREFIX}/include/third_party
+)
+
+# Library dependencies.
+set(PADDLE_LIBRARIES_DIRS ${PADDLE_INSTALL_PREFIX}/lib)
+link_directories(${PADDLE_LIBRARIES_DIRS})
+
+file(GLOB PADDLE_LIBRARIES ${PADDLE_LIBRARIES_DIRS}/lib*)
+
+find_package(PythonLibs @PY_VERSION@ REQUIRED)
+list(APPEND PADDLE_INCLUDE_DIRS ${PYTHON_INCLUDE_DIRS})
+list(APPEND PADDLE_LIBRARIES ${PYTHON_LIBRARIES})
+
+if(@WITH_GPU@)
+    find_package(CUDA @CUDA_VERSION@ REQUIRED)
+    list(APPEND PADDLE_LIBRARIES ${CUDA_LIBRARIES})
+endif()

cmake/external/cinn.cmake

@@ -20,7 +20,7 @@ if(NOT CINN_GIT_TAG)
   set(CINN_GIT_TAG develop)
 endif()
 
-message(STATUS "CINN version: " ${CINN_GIT_TAG})
+message(STATUS "CINN  version: " ${CINN_GIT_TAG})
 
 # TODO(zhhsplendid): CINN has lots of warnings during early development.
 # They will be treated as errors under paddle. We set no-error now and we will

cmake/external/cutlass.cmake

@@ -42,20 +42,33 @@ ExternalProject_Add(
   INSTALL_COMMAND ""
   TEST_COMMAND "")
 
+set(gemm_operations_file
+    ${CMAKE_SOURCE_DIR}/paddle/phi/kernels/sparse/gpu/cutlass_generator/generated/gemm/all_gemm_operations.h
+)
+set(configurations_file
+    ${CMAKE_SOURCE_DIR}/paddle/phi/kernels/sparse/gpu/cutlass_generator/generated/gemm/configurations.h
+)
+set(tmp_gemm_operations_file ${gemm_operations_file}.tmp)
+set(tmp_configurations_file ${configurations_file}.tmp)
+
 add_custom_target(
   cutlass_codegen
-  COMMAND
-    rm -rf
-    ${CMAKE_SOURCE_DIR}/paddle/phi/kernels/sparse/gpu/cutlass_generator/build
-  COMMAND
-    mkdir -p
-    ${CMAKE_SOURCE_DIR}/paddle/phi/kernels/sparse/gpu/cutlass_generator/build/generated/gemm
   COMMAND
     ${PYTHON_EXECUTABLE} -B
     ${CMAKE_SOURCE_DIR}/paddle/phi/kernels/sparse/gpu/cutlass_generator/gather_gemm_scatter_generator.py
     "${THIRD_PARTY_PATH}/cutlass/src/extern_cutlass/tools/library/scripts/"
-    "${CMAKE_SOURCE_DIR}/paddle/phi/kernels/sparse/gpu/cutlass_generator/build"
+    "${CMAKE_SOURCE_DIR}/paddle/phi/kernels/sparse/gpu/cutlass_generator"
     "${CMAKE_CUDA_COMPILER_VERSION}"
+  COMMAND ${CMAKE_COMMAND} -E copy_if_different ${tmp_gemm_operations_file}
+          ${gemm_operations_file}
+  COMMAND
+    ${CMAKE_COMMAND} -E echo
+    "copy_if_different ${tmp_gemm_operations_file} to ${gemm_operations_file}"
+  COMMAND ${CMAKE_COMMAND} -E copy_if_different ${tmp_configurations_file}
+          ${configurations_file}
+  COMMAND
+    ${CMAKE_COMMAND} -E echo
+    "copy_if_different ${tmp_configurations_file} to ${configurations_file}"
   VERBATIM)
 
 add_library(cutlass INTERFACE)

cmake/external/dlpack.cmake

@@ -17,6 +17,7 @@ include(ExternalProject)
 set(DLPACK_PREFIX_DIR ${THIRD_PARTY_PATH}/dlpack)
 
 set(DLPACK_REPOSITORY ${GIT_URL}/dmlc/dlpack.git)
+
 set(DLPACK_TAG v0.4)
 
 set(DLPACK_INCLUDE_DIR ${THIRD_PARTY_PATH}/dlpack/src/extern_dlpack/include)

cmake/external/eigen.cmake

@@ -51,9 +51,10 @@ if(CMAKE_COMPILER_IS_GNUCC)
   if(GCC_VERSION GREATER_EQUAL "12.0")
     file(TO_NATIVE_PATH ${PADDLE_SOURCE_DIR}/patches/eigen/Complex.h.patch
          complex_header)
+    # See: [Why calling some `git` commands before `patch`?]
     set(EIGEN_PATCH_COMMAND
-        patch -Nd ${EIGEN_SOURCE_DIR}/Eigen/src/Core/arch/SSE/ <
-        ${complex_header})
+        git checkout -- . && git checkout ${EIGEN_TAG} && patch -Nd
+        ${EIGEN_SOURCE_DIR}/Eigen/src/Core/arch/SSE/ < ${complex_header})
   endif()
 endif()
 

cmake/external/gloo.cmake

@@ -37,8 +37,9 @@ if(WITH_GPU)
                                                   VERSION_GREATER 12.0)
     file(TO_NATIVE_PATH ${PADDLE_SOURCE_DIR}/patches/gloo/device.cc.patch
          native_dst)
-    set(GLOO_PATCH_COMMAND patch -d ${GLOO_SOURCE_DIR}/gloo/transport/tcp <
-                           ${native_dst})
+    set(GLOO_PATCH_COMMAND
+        git checkout -- . && git checkout ${GLOO_TAG} &&patch -Nd
+        ${GLOO_SOURCE_DIR}/gloo/transport/tcp < ${native_dst})
   endif()
 endif()
 
@@ -54,9 +55,11 @@ if(CMAKE_COMPILER_IS_GNUCC)
          native_dst)
     file(TO_NATIVE_PATH ${PADDLE_SOURCE_DIR}/patches/gloo/types.h.patch
          types_header)
+    # See: [Why calling some `git` commands before `patch`?]
     set(GLOO_PATCH_COMMAND
-        patch -Nd ${GLOO_SOURCE_DIR}/gloo/transport/tcp < ${native_dst} &&
-        patch -Nd ${GLOO_SOURCE_DIR}/gloo/ < ${types_header})
+        git checkout -- . && git checkout ${GLOO_TAG} && patch -Nd
+        ${GLOO_SOURCE_DIR}/gloo/transport/tcp < ${native_dst} && patch -Nd
+        ${GLOO_SOURCE_DIR}/gloo/ < ${types_header})
   endif()
 endif()
 include_directories(${GLOO_INCLUDE_DIR})

cmake/external/gtest.cmake

@@ -66,8 +66,10 @@ endif()
 if(NOT WIN32 AND ${CMAKE_CXX_COMPILER_VERSION} VERSION_GREATER 12.0)
   file(TO_NATIVE_PATH
        ${PADDLE_SOURCE_DIR}/patches/gtest/gtest-death-test.cc.patch native_src)
-  set(GTEST_PATCH_COMMAND patch -Nd ${GTEST_SOURCE_DIR}/googletest/src <
-                          ${native_src})
+  # See: [Why calling some `git` commands before `patch`?]
+  set(GTEST_PATCH_COMMAND
+      git checkout -- . && git checkout ${GTEST_TAG} && patch -Nd
+      ${GTEST_SOURCE_DIR}/googletest/src < ${native_src})
 endif()
 if(WIN32)
   ExternalProject_Add(

cmake/external/pybind11.cmake

@@ -25,8 +25,13 @@ set(PYBIND_PATCH_COMMAND "")
 if(NOT WIN32)
   file(TO_NATIVE_PATH ${PADDLE_SOURCE_DIR}/patches/pybind/cast.h.patch
        native_dst)
-  set(PYBIND_PATCH_COMMAND patch -d ${PYBIND_INCLUDE_DIR}/pybind11 <
-                           ${native_dst})
+  # Note: [Why calling some `git` commands before `patch`?]
+  # Paddle's CI uses cache to accelarate the make process. However, error might raise when patch codes in two scenarios:
+  # 1. Patch to the wrong version: the tag version of CI's cache falls behind PYBIND_TAG, use `git checkout ${PYBIND_TAG}` to solve this.
+  # 2. Patch twice: the tag version of cache == PYBIND_TAG, but patch has already applied to cache.
+  set(PYBIND_PATCH_COMMAND
+      git checkout -- . && git checkout ${PYBIND_TAG} && patch -Nd
+      ${PYBIND_INCLUDE_DIR}/pybind11 < ${native_dst})
 endif()
 
 ExternalProject_Add(

cmake/external/xpu.cmake

@@ -8,7 +8,7 @@ set(XPU_API_LIB_NAME "libxpuapi.so")
 set(XPU_RT_LIB_NAME "libxpurt.so")
 set(XPU_XFT_LIB_NAME "libxft.so")
 
-set(XPU_BASE_DATE "20230408")
+set(XPU_BASE_DATE "20230427")
 set(XPU_XCCL_BASE_VERSION "1.0.13")
 set(XPU_XFT_BASE_VERSION "latest")
 

cmake/inference_lib.cmake

@@ -12,11 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-# make package for paddle fluid shared and static library
-set(PADDLE_INSTALL_DIR
-    "${CMAKE_BINARY_DIR}/paddle_install_dir"
-    CACHE STRING "A path setting paddle shared and static libraries")
-
+# make package for paddle inference shared and static library
 set(PADDLE_INFERENCE_INSTALL_DIR
     "${CMAKE_BINARY_DIR}/paddle_inference_install_dir"
     CACHE STRING "A path setting paddle inference shared and static libraries")
@@ -214,7 +210,7 @@ endfunction()
 # inference library for only inference
 set(inference_lib_deps third_party paddle_inference paddle_inference_c
                        paddle_inference_shared paddle_inference_c_shared)
-add_custom_target(inference_lib_dist DEPENDS ${inference_lib_deps})
+add_custom_target(inference_lib_dist ALL DEPENDS ${inference_lib_deps})
 
 set(dst_dir "${PADDLE_INFERENCE_INSTALL_DIR}/third_party/threadpool")
 copy(
@@ -387,158 +383,5 @@ if(WITH_STRIP AND NOT WIN32)
   )
 endif()
 
-# fluid library for both train and inference
-set(fluid_lib_deps inference_lib_dist)
-add_custom_target(fluid_lib_dist ALL DEPENDS ${fluid_lib_deps})
-
-set(dst_dir "${PADDLE_INSTALL_DIR}/paddle/fluid")
-set(module "inference")
-if(WIN32)
-  copy(
-    fluid_lib_dist
-    SRCS ${src_dir}/${module}/*.h ${src_dir}/${module}/api/paddle_*.h
-         ${paddle_inference_lib}
-    DSTS ${dst_dir}/${module} ${dst_dir}/${module} ${dst_dir}/${module}
-         ${dst_dir}/${module})
-else()
-  copy(
-    fluid_lib_dist
-    SRCS ${src_dir}/${module}/*.h ${src_dir}/${module}/api/paddle_*.h
-         ${paddle_inference_lib}
-    DSTS ${dst_dir}/${module} ${dst_dir}/${module} ${dst_dir}/${module})
-endif()
-
-set(module "framework")
-set(framework_lib_deps framework_proto data_feed_proto trainer_desc_proto)
-add_dependencies(fluid_lib_dist ${framework_lib_deps})
-copy(
-  fluid_lib_dist
-  SRCS ${src_dir}/${module}/*.h
-       ${src_dir}/${module}/details/*.h
-       ${PADDLE_BINARY_DIR}/paddle/fluid/framework/trainer_desc.pb.h
-       ${PADDLE_BINARY_DIR}/paddle/fluid/framework/framework.pb.h
-       ${PADDLE_BINARY_DIR}/paddle/fluid/framework/data_feed.pb.h
-       ${src_dir}/${module}/ir/memory_optimize_pass/*.h
-       ${src_dir}/${module}/ir/*.h
-       ${src_dir}/${module}/fleet/*.h
-  DSTS ${dst_dir}/${module}
-       ${dst_dir}/${module}/details
-       ${dst_dir}/${module}
-       ${dst_dir}/${module}
-       ${dst_dir}/${module}
-       ${dst_dir}/${module}/ir/memory_optimize_pass
-       ${dst_dir}/${module}/ir
-       ${dst_dir}/${module}/fleet)
-
-set(module "operators")
-copy(
-  fluid_lib_dist
-  SRCS ${src_dir}/${module}/reader/blocking_queue.h
-  DSTS ${dst_dir}/${module}/reader/)
-
-set(module "memory")
-copy(
-  fluid_lib_dist
-  SRCS ${src_dir}/${module}/allocation/*.h
-  DSTS ${dst_dir}/${module}/allocation)
-
-set(module "platform")
-set(platform_lib_deps phi_profiler_proto errors)
-if(WITH_GPU)
-  set(platform_lib_deps ${platform_lib_deps} external_error_proto)
-endif()
-
-add_dependencies(fluid_lib_dist ${platform_lib_deps})
-copy(
-  fluid_lib_dist
-  SRCS ${src_dir}/${module}/*.h ${src_dir}/${module}/dynload/*.h
-       ${PADDLE_BINARY_DIR}/paddle/phi/api/profiler/*.pb.h
-  DSTS ${dst_dir}/${module} ${dst_dir}/${module}/dynload ${dst_dir}/${module})
-
-set(module "string")
-copy(
-  fluid_lib_dist
-  SRCS ${PADDLE_SOURCE_DIR}/paddle/utils/${module}/*.h
-       ${PADDLE_SOURCE_DIR}/paddle/utils/${module}/tinyformat/*.h
-  DSTS ${dst_dir}/${module} ${dst_dir}/${module}/tinyformat)
-
-set(module "imperative")
-copy(
-  fluid_lib_dist
-  SRCS ${src_dir}/${module}/*.h ${src_dir}/${module}/jit/*.h
-  DSTS ${dst_dir}/${module} ${dst_dir}/${module}/jit)
-
-set(module "pybind")
-copy(
-  fluid_lib_dist
-  SRCS ${CMAKE_CURRENT_BINARY_DIR}/paddle/fluid/${module}/pybind.h
-  DSTS ${dst_dir}/${module})
-
-set(dst_dir "${PADDLE_INSTALL_DIR}/third_party/eigen3")
-copy(
-  inference_lib_dist
-  SRCS ${EIGEN_INCLUDE_DIR}/Eigen/Core ${EIGEN_INCLUDE_DIR}/Eigen/src
-       ${EIGEN_INCLUDE_DIR}/unsupported/Eigen
-  DSTS ${dst_dir}/Eigen ${dst_dir}/Eigen ${dst_dir}/unsupported)
-
-set(dst_dir "${PADDLE_INSTALL_DIR}/third_party/dlpack")
-copy(
-  inference_lib_dist
-  SRCS ${DLPACK_INCLUDE_DIR}/dlpack
-  DSTS ${dst_dir})
-
-set(dst_dir "${PADDLE_INSTALL_DIR}/third_party/install/zlib")
-copy(
-  inference_lib_dist
-  SRCS ${ZLIB_INCLUDE_DIR} ${ZLIB_LIBRARIES}
-  DSTS ${dst_dir} ${dst_dir}/lib)
-
-# CMakeCache Info
-copy(
-  fluid_lib_dist
-  SRCS ${PADDLE_INFERENCE_INSTALL_DIR}/third_party
-       ${CMAKE_CURRENT_BINARY_DIR}/CMakeCache.txt
-  DSTS ${PADDLE_INSTALL_DIR} ${PADDLE_INSTALL_DIR})
-
-# paddle fluid version
-function(version version_file)
-  execute_process(
-    COMMAND ${GIT_EXECUTABLE} log --pretty=format:%H -1
-    WORKING_DIRECTORY ${PADDLE_SOURCE_DIR}
-    OUTPUT_VARIABLE PADDLE_GIT_COMMIT)
-  file(
-    WRITE ${version_file}
-    "GIT COMMIT ID: ${PADDLE_GIT_COMMIT}\n" "WITH_MKL: ${WITH_MKL}\n"
-    "WITH_MKLDNN: ${WITH_MKLDNN}\n" "WITH_GPU: ${WITH_GPU}\n"
-    "WITH_ROCM: ${WITH_ROCM}\n" "WITH_IPU: ${WITH_IPU}\n")
-  if(WITH_GPU)
-    file(APPEND ${version_file}
-         "CUDA version: ${CUDA_VERSION}\n"
-         "CUDNN version: v${CUDNN_MAJOR_VERSION}.${CUDNN_MINOR_VERSION}\n")
-  endif()
-  if(WITH_ROCM)
-    file(APPEND ${version_file}
-         "HIP version: v${HIP_MAJOR_VERSION}.${HIP_MINOR_VERSION}\n"
-         "MIOpen version: v${MIOPEN_MAJOR_VERSION}.${MIOPEN_MINOR_VERSION}\n")
-  endif()
-  if(WITH_IPU)
-    file(APPEND ${version_file} "PopART version: ${POPART_VERSION}\n")
-  endif()
-  file(APPEND ${version_file}
-       "CXX compiler version: ${CMAKE_CXX_COMPILER_VERSION}\n")
-  if(TENSORRT_FOUND)
-    file(
-      APPEND ${version_file}
-      "WITH_TENSORRT: ${TENSORRT_FOUND}\n"
-      "TensorRT version: v${TENSORRT_MAJOR_VERSION}.${TENSORRT_MINOR_VERSION}.${TENSORRT_PATCH_VERSION}.${TENSORRT_BUILD_VERSION}\n"
-    )
-  endif()
-  if(WITH_LITE)
-    file(APPEND ${version_file} "WITH_LITE: ${WITH_LITE}\n"
-                                "LITE_GIT_TAG: ${LITE_GIT_TAG}\n")
-  endif()
-
-endfunction()
-version(${PADDLE_INSTALL_DIR}/version.txt)
 version(${PADDLE_INFERENCE_INSTALL_DIR}/version.txt)
 version(${PADDLE_INFERENCE_C_INSTALL_DIR}/version.txt)