Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

add go_xxx to simplify cmake #2182

Merged
merged 13 commits into from
May 20, 2017
Merged

add go_xxx to simplify cmake #2182

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
13 commits
Select commit Hold shift + click to select a range
c9cdc98
detect golang in cmake
gangliao May 17, 2017
4dcb9f1
add go_xxx to simplify cmake
gangliao May 17, 2017
e254c7a
add go unit test
gangliao May 17, 2017
4d5417b
modify travis
gangliao May 17, 2017
4471452
update travis
gangliao May 17, 2017
af06519
clang-format test.cc
gangliao May 17, 2017
00441a9
fix conficts
gangliao May 17, 2017
a7edafc
add cc_test
gangliao May 17, 2017
589cea1
add $ENV{GOROOT}/bin
gangliao May 17, 2017
071c65f
add go extern
gangliao May 17, 2017
4f837b9
add ${CMAKE_THREAD_LIBS_INIT}
gangliao May 18, 2017
5b32283
fix conflicts
gangliao May 18, 2017
50d0e26
pass travis ci
gangliao May 20, 2017
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion CMakeLists.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ set(PROJ_ROOT ${CMAKE_CURRENT_SOURCE_DIR})

include(system)

project(paddle CXX C)
project(paddle CXX C Go)

find_package(Sphinx)
if(NOT CMAKE_CROSSCOMPILING)
Expand Down
46 changes: 46 additions & 0 deletions cmake/CMakeDetermineGoCompiler.cmake
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,46 @@
if(NOT CMAKE_Go_COMPILER)
if(NOT $ENV{GO_COMPILER} STREQUAL "")
get_filename_component(CMAKE_Go_COMPILER_INIT $ENV{GO_COMPILER} PROGRAM PROGRAM_ARGS CMAKE_Go_FLAGS_ENV_INIT)

if(CMAKE_Go_FLAGS_ENV_INIT)
set(CMAKE_Go_COMPILER_ARG1 "${CMAKE_Go_FLAGS_ENV_INIT}" CACHE STRING "First argument to Go compiler")
endif()

if(NOT EXISTS ${CMAKE_Go_COMPILER_INIT})
message(SEND_ERROR "Could not find compiler set in environment variable GO_COMPILER:\n$ENV{GO_COMPILER}.")
endif()

endif()

set(Go_BIN_PATH
$ENV{GOPATH}
$ENV{GOROOT}
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

go bin path 一般在$ENV{GOROOT}/bin

Copy link
Contributor Author

@gangliao gangliao May 17, 2017 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

done

$ENV{GOROOT}/bin
$ENV{GO_COMPILER}
/usr/bin
/usr/local/bin
)

if(CMAKE_Go_COMPILER_INIT)
set(CMAKE_Go_COMPILER ${CMAKE_Go_COMPILER_INIT} CACHE PATH "Go Compiler")
else()
find_program(CMAKE_Go_COMPILER
NAMES go
PATHS ${Go_BIN_PATH}
)
if(CMAKE_Go_COMPILER)
EXEC_PROGRAM(${CMAKE_Go_COMPILER} ARGS version OUTPUT_VARIABLE GOLANG_VERSION)
STRING(REGEX MATCH "go[0-9]+[.0-9]*[ /A-Za-z0-9]*" VERSION "${GOLANG_VERSION}")
message("-- The Golang compiler identification is ${VERSION}")
message("-- Check for working Golang compiler: ${CMAKE_Go_COMPILER}")
endif()
endif()

endif()

mark_as_advanced(CMAKE_Go_COMPILER)

configure_file(${CMAKE_CURRENT_SOURCE_DIR}/cmake/CMakeGoCompiler.cmake.in
${CMAKE_PLATFORM_INFO_DIR}/CMakeGoCompiler.cmake @ONLY)

set(CMAKE_Go_COMPILER_ENV_VAR "GO_COMPILER")
8 changes: 8 additions & 0 deletions cmake/CMakeGoCompiler.cmake.in
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
set(CMAKE_Go_COMPILER "@CMAKE_Go_COMPILER@")
set(CMAKE_Go_COMPILER_LOADED 1)

set(CMAKE_Go_SOURCE_FILE_EXTENSIONS go)
set(CMAKE_Go_LINKER_PREFERENCE 40)
set(CMAKE_Go_OUTPUT_EXTENSION .o)
set(CMAKE_Go_OUTPUT_EXTENSION_REPLACE 1)
set(CMAKE_Go_COMPILER_ENV_VAR "GO_COMPILER")
7 changes: 7 additions & 0 deletions cmake/CMakeGoInformation.cmake
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
if(NOT CMAKE_Go_COMPILE_OBJECT)
set(CMAKE_Go_COMPILE_OBJECT "go tool compile -l -N -o <OBJECT> <SOURCE> ")
endif()

if(NOT CMAKE_Go_LINK_EXECUTABLE)
set(CMAKE_Go_LINK_EXECUTABLE "go tool link -o <TARGET> <OBJECTS> ")
endif()
1 change: 1 addition & 0 deletions cmake/CMakeTestGoCompiler.cmake
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
set(CMAKE_Go_COMPILER_WORKS 1 CACHE INTERNAL "")
76 changes: 76 additions & 0 deletions cmake/generic.cmake
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@
#
# cmake_parse_arguments can help us to achieve this goal.
# https://cmake.org/cmake/help/v3.0/module/CMakeParseArguments.html
#

# cc_library parses tensor.cc and figures out that target also depend on tensor.h.
# cc_library(tensor
Expand Down Expand Up @@ -139,3 +140,78 @@ function(nv_test TARGET_NAME)
endif()
add_test(${TARGET_NAME} ${TARGET_NAME})
endfunction(nv_test)

set(GOPATH "${CMAKE_CURRENT_BINARY_DIR}/go")
file(MAKE_DIRECTORY ${GOPATH})

# Because api.go defines a GO wrapper to ops and tensor, it depends on
# both. This implies that if any of tensor.{h,cc}, ops.{h,cu}, or
# api.go is changed, api need to be re-built.
# go_library(api
# SRCS
# api.go
# DEPS
# tensor # Because ops depend on tensor, this line is optional.
# ops)
function(go_library TARGET_NAME)
set(options OPTIONAL)
set(oneValueArgs "")
set(multiValueArgs SRCS DEPS)
cmake_parse_arguments(go_library "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN})
if (${go_library_OPTIONAL} STREQUAL "SHARED")
set(BUILD_MODE "-buildmode=c-shared")
if(APPLE)
set(LIB_NAME "lib${TARGET_NAME}.dylib")
else()
set(LIB_NAME "lib${TARGET_NAME}.so")
endif()
else()
set(BUILD_MODE "-buildmode=c-archive")
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think we might have to support more types of work in addition to c-shared and c-archive in go_library? These two types are Go libraries that can be called from C, and there are special requirements with the source code -- e.g., all exported symbols must be in package main, and there must be a main function defined. I think most Go libraries are supposed to be called only from Go code, and should be in some other type -- archive maybe? @helinwang

Copy link
Contributor

@helinwang helinwang May 17, 2017 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@wangkuiyi Good question! You mean how to specify Go dependencies when using go_binary? I think it would be painful to specify Go dependencies manually in cmake (unlike C++ needs to use cmake or other tools for dependency management, Go already have it's dependency management tool). Maybe in cmake we can just let the go tool do it for us:

export GOPATH=./build/go
go get .path_to_go_binary_project/...

in this way, go_library is only for the library used by other languages.
reference: http://stackoverflow.com/a/30296041/852385

Copy link
Collaborator

@wangkuiyi wangkuiyi May 17, 2017 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

What I was talking about is that go_library should be able to generate the following types of outputs:

  1. A static library which is callable from other Go code. To build it, we need something like the following:

    go build \
-o ~/work/paddle/build/libpaddle_go_pserver_libpserver.a \ 
~/work/paddle/paddle/go/pserver/pserver.go

    Please be aware of the correspondence between source file path and output archive file path.

  2. A static library which is callable from C/C++. We need something like

    go build -buildmode=c-archive \
-o ~/work/paddle/build/libpaddle_go_pserver_cclient_cclient.a
~/work/paddle/paddle/go/pserver/cclient/cclient.go

  3. A dynamic library which is callable from C/C++.

    go build -buildmode=c-shared \
-o ~/work/paddle/build/libpaddle_go_pserver_cclient_cclient.so
~/work/paddle/paddle/go/pserver/cclient/cclient.go

set(LIB_NAME "lib${TARGET_NAME}.a")
endif()
add_custom_command(OUTPUT ${TARGET_NAME}_timestamp
COMMAND env GOPATH=${GOPATH} ${CMAKE_Go_COMPILER} build ${BUILD_MODE}
-o "${CMAKE_CURRENT_BINARY_DIR}/${LIB_NAME}"
${go_library_SRCS}
WORKING_DIRECTORY ${CMAKE_CURRENT_LIST_DIR})
add_custom_target(${TARGET_NAME}_lib ALL DEPENDS ${TARGET_NAME}_timestamp ${go_library_DEPS})
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Have not found where go_library_DEPS is set?
Also please consider let the Go tool to get Go dependency rather than manually specifying it in cmake: https://github.com/PaddlePaddle/Paddle/pull/2182/files/af065196400ca0254e61df3916888f4341306e97#r117078644

add_library(${TARGET_NAME} STATIC IMPORTED)
set_property(TARGET ${TARGET_NAME} PROPERTY
IMPORTED_LOCATION "${CMAKE_CURRENT_BINARY_DIR}/${LIB_NAME}")
add_dependencies(${TARGET_NAME} ${TARGET_NAME}_lib)
endfunction(go_library)

function(go_binary TARGET_NAME)
set(options OPTIONAL)
set(oneValueArgs "")
set(multiValueArgs SRCS DEPS)
cmake_parse_arguments(go_binary "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN})
add_custom_command(OUTPUT ${TARGET_NAME}_timestamp
COMMAND env GOPATH=${GOPATH} ${CMAKE_Go_COMPILER} build
-o "${CMAKE_CURRENT_BINARY_DIR}/${TARGET_NAME}"
${go_library_SRCS}
WORKING_DIRECTORY ${CMAKE_CURRENT_LIST_DIR})
add_custom_target(${TARGET_NAME} ALL DEPENDS ${TARGET_NAME}_timestamp ${go_binary_DEPS})
install(PROGRAMS ${CMAKE_CURRENT_BINARY_DIR}/${TARGET_NAME} DESTINATION bin)
endfunction(go_binary)

function(go_test TARGET_NAME)
set(options OPTIONAL)
set(oneValueArgs "")
set(multiValueArgs SRCS DEPS)
cmake_parse_arguments(go_test "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN})
add_custom_command(OUTPUT ${TARGET_NAME}_timestamp
COMMAND env GOPATH=${GOPATH} ${CMAKE_Go_COMPILER} test
-c -o "${CMAKE_CURRENT_BINARY_DIR}/${TARGET_NAME}"
${go_test_SRCS}
WORKING_DIRECTORY ${CMAKE_CURRENT_LIST_DIR})
add_custom_target(${TARGET_NAME} ALL DEPENDS ${TARGET_NAME}_timestamp ${go_test_DEPS})
add_test(${TARGET_NAME} ${CMAKE_CURRENT_BINARY_DIR}/${TARGET_NAME})
endfunction(go_test)

# go_extern will download extern go project.
# go_extern(target_name extern_source)
# go_extern(go_redis github.com/hoisie/redis)
function(go_extern TARGET_NAME)
add_custom_target(${TARGET_NAME} env GOPATH=${GOPATH} ${CMAKE_Go_COMPILER} get ${ARGN})
endfunction(go_extern)
4 changes: 4 additions & 0 deletions paddle/CMakeLists.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,10 @@ add_subdirectory(pserver)
add_subdirectory(trainer)
add_subdirectory(scripts)

if(CMAKE_Go_COMPILER)
add_subdirectory(go)
endif()

find_package(Boost QUIET)

if(Boost_FOUND)
Expand Down
9 changes: 9 additions & 0 deletions paddle/go/CMakeLists.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
include_directories(${CMAKE_CURRENT_BINARY_DIR})

go_library(adder SRCS adder.go)

cc_test(cgo_test
SRCS
cgo_test.cc
DEPS
adder)
10 changes: 10 additions & 0 deletions paddle/go/adder.go
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
package main

import "C"

//export GoAdder
func GoAdder(x, y int) int {
return x + y
}

func main() {} // Required but ignored
5 changes: 5 additions & 0 deletions paddle/go/cgo_test.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
#include <iostream>
#include "gtest/gtest.h"
#include "libadder.h"

TEST(Cgo, Invoke) { EXPECT_EQ(GoAdder(30, 12), 42); }
Copy link
Contributor

@helinwang helinwang May 17, 2017 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

For the linking of this file's executable to work, we will need set (CMAKE_EXE_LINKER_FLAGS "-pthread"), because when linking a binary that contains Go code built with cgo enabled, pthread need to be linked. (I think now CI is failing because of this).

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

done