Skip to content

Commit 6a7f121

Browse files
committed
Merge branch 'develop' of https://github.com/PaddlePaddle/Paddle into row_conv_fix
2 parents e287034 + ddc0c21 commit 6a7f121

File tree

13 files changed

+107
-253
lines changed

13 files changed

+107
-253
lines changed

CMakeLists.txt

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -130,14 +130,16 @@ if(WITH_GPU)
130130
endif(WITH_GPU)
131131

132132
add_subdirectory(proto)
133-
add_subdirectory(paddle)
134-
add_subdirectory(python)
135133

134+
# "add_subdirectory(paddle)" and "add_subdirectory(python)" should be
135+
# placed after this block, because they depends on it.
136136
if(WITH_GOLANG)
137-
#TODO (add go/master/c back when fixed)
138-
add_subdirectory(go/pserver/cclient)
137+
add_subdirectory(go/master/c)
138+
add_subdirectory(go/pserver/cclient)
139139
endif(WITH_GOLANG)
140140

141+
add_subdirectory(paddle)
142+
add_subdirectory(python)
141143
if(WITH_DOC)
142144
add_subdirectory(doc)
143145
endif()

cmake/configure.cmake

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -69,3 +69,27 @@ endif(NOT WITH_GPU)
6969

7070
set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${SIMD_FLAG}")
7171
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${SIMD_FLAG}")
72+
73+
if(WITH_GOLANG)
74+
# we need to symlink Paddle directory into GOPATH. If we
75+
# don't do it and we have code that depends on Paddle, go
76+
# get ./... will download a new Paddle repo from Github,
77+
# without the changes in our current Paddle repo that we
78+
# want to build.
79+
set(GOPATH "${CMAKE_CURRENT_BINARY_DIR}/go")
80+
file(MAKE_DIRECTORY ${GOPATH})
81+
set(PADDLE_IN_GOPATH "${GOPATH}/src/github.com/PaddlePaddle/Paddle")
82+
add_custom_target(go_path)
83+
add_custom_command(TARGET go_path
84+
# Symlink Paddle directory into GOPATH
85+
COMMAND mkdir -p ${PADDLE_IN_GOPATH}
86+
COMMAND rm -rf ${PADDLE_IN_GOPATH}
87+
COMMAND ln -sf ${CMAKE_SOURCE_DIR} ${PADDLE_IN_GOPATH}
88+
# Automatically get all dependencies specified in the source code
89+
# We can't run `go get -d ./...` for every target, because
90+
# multiple `go get` can not run concurrently, but make need to be
91+
# able to run with multiple jobs.
92+
COMMAND env GOPATH=${GOPATH} ${CMAKE_Go_COMPILER} get -d ./go/...
93+
WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
94+
)
95+
endif(WITH_GOLANG)

cmake/external/protobuf.cmake

Lines changed: 1 addition & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -144,18 +144,7 @@ FUNCTION(build_protobuf TARGET_NAME BUILD_FOR_HOST)
144144
ENDFUNCTION()
145145

146146
SET(PROTOBUF_VERSION 3.1)
147-
IF(NOT CMAKE_CROSSCOMPILING)
148-
FIND_PACKAGE(Protobuf ${PROTOBUF_VERSION})
149-
150-
IF(PROTOBUF_FOUND)
151-
SET_PROTOBUF_VERSION()
152-
IF("${PROTOBUF_VERSION}" VERSION_LESS "3.1.0")
153-
SET(PROTOBUF_FOUND OFF)
154-
ELSE()
155-
PROMPT_PROTOBUF_LIB()
156-
ENDIF()
157-
ENDIF(PROTOBUF_FOUND)
158-
ELSE()
147+
IF(CMAKE_CROSSCOMPILING)
159148
build_protobuf(protobuf_host TRUE)
160149
LIST(APPEND external_project_dependencies protobuf_host)
161150

cmake/external/python.cmake

Lines changed: 0 additions & 187 deletions
Original file line numberDiff line numberDiff line change
@@ -32,193 +32,6 @@ IF(PYTHONINTERP_FOUND)
3232
MESSAGE(FATAL_ERROR "Found Python Protobuf ${PY_GOOGLE.PROTOBUF_VERSION} < 3.0.0, "
3333
"please use pip to upgrade protobuf. pip install -U protobuf")
3434
ENDIF()
35-
ELSE(PYTHONINTERP_FOUND)
36-
MESSAGE(FATAL_ERROR "Please install python 2.7 before building PaddlePaddle.")
37-
##################################### PYTHON ########################################
38-
SET(PYTHON_SOURCES_DIR ${THIRD_PARTY_PATH}/python)
39-
SET(PYTHON_INSTALL_DIR ${THIRD_PARTY_PATH}/install/python)
40-
SET(_python_DIR ${PYTHON_INSTALL_DIR})
41-
42-
IF(UNIX)
43-
SET(PYTHON_FOUND ON)
44-
SET(PYTHON_INCLUDE_DIR "${PYTHON_INSTALL_DIR}/include/python2.7" CACHE PATH "Python include dir" FORCE)
45-
SET(PYTHON_LIBRARIES "${PYTHON_INSTALL_DIR}/lib/libpython2.7.a" CACHE FILEPATH "Python library" FORCE)
46-
SET(PYTHON_EXECUTABLE ${PYTHON_INSTALL_DIR}/bin/python CACHE FILEPATH "Python executable" FORCE)
47-
SET(PY_SITE_PACKAGES_PATH "${PYTHON_INSTALL_DIR}/lib/python2.7/site-packages" CACHE PATH "Python site-packages path" FORCE)
48-
ELSEIF(WIN32)
49-
SET(PYTHON_FOUND ON)
50-
SET(PYTHON_INCLUDE_DIR "${PYTHON_INSTALL_DIR}/include" CACHE PATH "Python include dir" FORCE)
51-
SET(PYTHON_LIBRARIES "${PYTHON_INSTALL_DIR}/libs/python27.lib" CACHE FILEPATH "Python library" FORCE)
52-
SET(PYTHON_EXECUTABLE "${PYTHON_INSTALL_DIR}/bin/python.exe" CACHE FILEPATH "Python executable" FORCE)
53-
SET(PY_SITE_PACKAGES_PATH "${PYTHON_INSTALL_DIR}/Lib/site-packages" CACHE PATH "Python site-packages path" FORCE)
54-
ELSE()
55-
MESSAGE(FATAL_ERROR "Unknown system !")
56-
ENDIF()
57-
58-
IF(APPLE)
59-
LIST(APPEND EXTERNAL_PROJECT_OPTIONAL_CMAKE_ARGS
60-
-DCMAKE_BUILD_WITH_INSTALL_RPATH:BOOL=ON
61-
)
62-
ENDIF()
63-
64-
SET(EXTERNAL_PROJECT_OPTIONAL_CMAKE_CACHE_ARGS)
65-
66-
# Force Python build to "Release".
67-
IF(CMAKE_CONFIGURATION_TYPES)
68-
SET(SAVED_CMAKE_CFG_INTDIR ${CMAKE_CFG_INTDIR})
69-
SET(CMAKE_CFG_INTDIR "Release")
70-
ELSE()
71-
LIST(APPEND EXTERNAL_PROJECT_OPTIONAL_CMAKE_CACHE_ARGS
72-
-DCMAKE_BUILD_TYPE:STRING=Release
73-
)
74-
ENDIF()
75-
76-
ExternalProject_Add(python
77-
${EXTERNAL_PROJECT_LOG_ARGS}
78-
GIT_REPOSITORY "https://github.com/python-cmake-buildsystem/python-cmake-buildsystem.git"
79-
PREFIX ${PYTHON_SOURCES_DIR}
80-
UPDATE_COMMAND ""
81-
CMAKE_ARGS -DPYTHON_VERSION=2.7.12
82-
CMAKE_ARGS -DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER}
83-
CMAKE_ARGS -DCMAKE_C_COMPILER=${CMAKE_C_COMPILER}
84-
CMAKE_CACHE_ARGS
85-
-DCMAKE_INSTALL_PREFIX:PATH=${PYTHON_INSTALL_DIR}
86-
-DBUILD_LIBPYTHON_SHARED:BOOL=OFF
87-
-DUSE_SYSTEM_LIBRARIES:BOOL=OFF
88-
-DZLIB_ROOT:FILEPATH=${ZLIB_ROOT}
89-
-DZLIB_INCLUDE_DIR:PATH=${ZLIB_INCLUDE_DIR}
90-
-DZLIB_LIBRARY:FILEPATH=${ZLIB_LIBRARIES}
91-
-DDOWNLOAD_SOURCES:BOOL=ON
92-
-DINSTALL_WINDOWS_TRADITIONAL:BOOL=OFF
93-
${EXTERNAL_PROJECT_OPTIONAL_CMAKE_CACHE_ARGS}
94-
${EXTERNAL_PROJECT_OPTIONAL_CMAKE_ARGS}
95-
DEPENDS zlib
96-
)
97-
98-
SET(py_env
99-
PATH=${PYTHON_INSTALL_DIR}/bin
100-
PYTHONHOME=${PYTHON_INSTALL_DIR}
101-
PYTHONPATH=${PYTHON_INSTALL_DIR}/lib:${PYTHON_INSTALL_DIR}/lib/python2.7:${PY_SITE_PACKAGES_PATH})
102-
####################################################################################
103-
104-
##################################### SETUPTOOLS ###################################
105-
SET(SETUPTOOLS_SOURCES_DIR ${PYTHON_SOURCES_DIR}/setuptools)
106-
ExternalProject_Add(setuptools
107-
${EXTERNAL_PROJECT_LOG_ARGS}
108-
PREFIX ${SETUPTOOLS_SOURCES_DIR}
109-
URL "https://pypi.python.org/packages/source/s/setuptools/setuptools-18.3.2.tar.gz"
110-
BUILD_IN_SOURCE 1
111-
PATCH_COMMAND ""
112-
UPDATE_COMMAND ""
113-
CONFIGURE_COMMAND ""
114-
INSTALL_COMMAND ""
115-
BUILD_COMMAND env ${py_env} ${PYTHON_EXECUTABLE} setup.py install
116-
DEPENDS python zlib
117-
)
118-
#####################################################################################
119-
120-
##################################### SIX ###########################################
121-
SET(SIX_SOURCES_DIR ${PYTHON_SOURCES_DIR}/six)
122-
ExternalProject_Add(six
123-
${EXTERNAL_PROJECT_LOG_ARGS}
124-
PREFIX ${SIX_SOURCES_DIR}
125-
URL https://pypi.python.org/packages/source/s/six/six-1.10.0.tar.gz
126-
BUILD_IN_SOURCE 1
127-
PATCH_COMMAND ""
128-
UPDATE_COMMAND ""
129-
CONFIGURE_COMMAND ""
130-
INSTALL_COMMAND ""
131-
BUILD_COMMAND env ${py_env} ${PYTHON_EXECUTABLE} setup.py install
132-
DEPENDS python setuptools
133-
)
134-
#####################################################################################
135-
136-
##################################### CYTHON ########################################
137-
SET(CYTHON_SOURCES_DIR ${PYTHON_SOURCES_DIR}/cython)
138-
ExternalProject_Add(cython
139-
${EXTERNAL_PROJECT_LOG_ARGS}
140-
PREFIX ${CYTHON_SOURCES_DIR}
141-
URL https://github.com/cython/cython/archive/0.25.2.tar.gz
142-
GIT_TAG 0.25.2
143-
BUILD_IN_SOURCE 1
144-
CONFIGURE_COMMAND ""
145-
PATCH_COMMAND ""
146-
UPDATE_COMMAND ""
147-
INSTALL_COMMAND ""
148-
BUILD_COMMAND env ${py_env} ${PYTHON_EXECUTABLE} setup.py install
149-
DEPENDS python
150-
)
151-
####################################################################################
152-
153-
##################################### NUMPY ########################################
154-
SET(NUMPY_SOURCES_DIR ${PYTHON_SOURCES_DIR}/numpy)
155-
SET(NUMPY_TAG_VERSION "v1.11.3")
156-
SET(NUMPY_VERSION "1.11.3")
157-
158-
SET(EGG_NAME "")
159-
SET(PYTHON_NUMPY_INCLUDE_DIR "")
160-
IF(WIN32)
161-
SET(EGG_NAME "numpy-${NUMPY_VERSION}-py2.7-${HOST_SYSTEM}.egg")
162-
ELSE(WIN32)
163-
IF(APPLE)
164-
SET(EGG_NAME "numpy-${NUMPY_VERSION}-py2.7-${HOST_SYSTEM}-${MACOS_VERSION}")
165-
ELSE(APPLE)
166-
SET(EGG_NAME "numpy-${NUMPY_VERSION}-py2.7-linux")
167-
SET(EGG_NAME "numpy-${NUMPY_VERSION}-py2.7-linux")
168-
ENDIF(APPLE)
169-
170-
FOREACH(suffix x86_64 intel fat64 fat32 universal)
171-
LIST(APPEND PYTHON_NUMPY_INCLUDE_DIR ${PY_SITE_PACKAGES_PATH}/${EGG_NAME}-${suffix}.egg/numpy/core/include)
172-
ENDFOREACH()
173-
ENDIF(WIN32)
174-
175-
ExternalProject_Add(numpy
176-
${EXTERNAL_PROJECT_LOG_ARGS}
177-
GIT_REPOSITORY https://github.com/numpy/numpy.git
178-
GIT_TAG ${NUMPY_TAG_VERSION}
179-
CONFIGURE_COMMAND ""
180-
UPDATE_COMMAND ""
181-
PREFIX ${NUMPY_SOURCES_DIR}
182-
BUILD_COMMAND env ${py_env} ${PYTHON_EXECUTABLE} setup.py build
183-
INSTALL_COMMAND env ${py_env} ${PYTHON_EXECUTABLE} setup.py install
184-
BUILD_IN_SOURCE 1
185-
DEPENDS python setuptools cython
186-
)
187-
####################################################################################
188-
189-
##################################### WHEEL ########################################
190-
SET(WHEEL_SOURCES_DIR ${PYTHON_SOURCES_DIR}/wheel)
191-
ExternalProject_Add(wheel
192-
${EXTERNAL_PROJECT_LOG_ARGS}
193-
URL https://pypi.python.org/packages/source/w/wheel/wheel-0.29.0.tar.gz
194-
PREFIX ${WHEEL_SOURCES_DIR}
195-
CONFIGURE_COMMAND ""
196-
UPDATE_COMMAND ""
197-
BUILD_COMMAND ""
198-
INSTALL_COMMAND env ${py_env} ${PYTHON_EXECUTABLE} setup.py install
199-
BUILD_IN_SOURCE 1
200-
DEPENDS python setuptools
201-
)
202-
####################################################################################
203-
204-
################################### PROTOBUF #######################################
205-
SET(PY_PROTOBUF_SOURCES_DIR ${PYTHON_SOURCES_DIR}/protobuf)
206-
ExternalProject_Add(python-protobuf
207-
${EXTERNAL_PROJECT_LOG_ARGS}
208-
URL https://pypi.python.org/packages/e0/b0/0a1b364fe8a7d177b4b7d4dca5b798500dc57a7273b93cca73931b305a6a/protobuf-3.1.0.post1.tar.gz
209-
URL_MD5 38b5fb160c768d2f8444d0c6d637ff91
210-
PREFIX ${PY_PROTOBUF_SOURCES_DIR}
211-
BUILD_IN_SOURCE 1
212-
PATCH_COMMAND ""
213-
CONFIGURE_COMMAND ""
214-
BUILD_COMMAND env ${py_env} ${PYTHON_EXECUTABLE} setup.py build
215-
INSTALL_COMMAND env ${py_env} ${PYTHON_EXECUTABLE} setup.py install
216-
DEPENDS python setuptools six
217-
)
218-
####################################################################################
219-
220-
LIST(APPEND external_project_dependencies python setuptools six cython wheel python-protobuf numpy)
221-
22235
ENDIF(PYTHONINTERP_FOUND)
22336

22437
IF(WITH_PYTHON)

cmake/generic.cmake

Lines changed: 10 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -146,9 +146,9 @@ function(merge_static_libs TARGET_NAME)
146146
set(outlibfile "$<TARGET_FILE:${TARGET_NAME}>")
147147

148148
foreach(lib ${libs})
149-
add_custom_command(TARGET ${TARGET_NAME} POST_BUILD
150-
COMMAND ${CMAKE_AR} ru ${outlibfile} @"../${objlistfile}"
151-
WORKING_DIRECTORY ${objdir})
149+
add_custom_command(TARGET ${TARGET_NAME} POST_BUILD
150+
COMMAND ${CMAKE_AR} ru ${outlibfile} @"../${lib}.objlist"
151+
WORKING_DIRECTORY ${lib}.objdir)
152152
endforeach()
153153

154154
add_custom_command(TARGET ${TARGET_NAME} POST_BUILD
@@ -256,10 +256,6 @@ function(nv_test TARGET_NAME)
256256
endif()
257257
endfunction(nv_test)
258258

259-
set(GOPATH "${CMAKE_CURRENT_BINARY_DIR}/go")
260-
file(MAKE_DIRECTORY ${GOPATH})
261-
set(PADDLE_IN_GOPATH "${GOPATH}/src/github.com/PaddlePaddle/Paddle")
262-
263259
function(go_library TARGET_NAME)
264260
set(options STATIC static SHARED shared)
265261
set(oneValueArgs "")
@@ -268,10 +264,10 @@ function(go_library TARGET_NAME)
268264

269265
if (go_library_SHARED OR go_library_shared)
270266
set(BUILD_MODE "-buildmode=c-shared")
271-
set(LIB_NAME "${CMAKE_SHARED_LIBRARY_PREFIX}${TARGET_NAME}${CMAKE_SHARED_LIBRARY_SUFFIX}")
267+
set(${TARGET_NAME}_LIB_NAME "${CMAKE_SHARED_LIBRARY_PREFIX}${TARGET_NAME}${CMAKE_SHARED_LIBRARY_SUFFIX}" CACHE STRING "output library name for target ${TARGET_NAME}")
272268
else()
273269
set(BUILD_MODE "-buildmode=c-archive")
274-
set(LIB_NAME "${CMAKE_STATIC_LIBRARY_PREFIX}${TARGET_NAME}${CMAKE_STATIC_LIBRARY_SUFFIX}")
270+
set(${TARGET_NAME}_LIB_NAME "${CMAKE_STATIC_LIBRARY_PREFIX}${TARGET_NAME}${CMAKE_STATIC_LIBRARY_SUFFIX}" CACHE STRING "output library name for target ${TARGET_NAME}")
275271
endif()
276272

277273
# Add dummy code to support `make target_name` under Terminal Command
@@ -286,25 +282,17 @@ function(go_library TARGET_NAME)
286282
add_dependencies(${TARGET_NAME} ${go_library_DEPS})
287283
endif(go_library_DEPS)
288284

289-
# we need to symlink Paddle directory into GOPATH. If we
290-
# don't do it and we have code that depends on Paddle, go
291-
# get ./... will download a new Paddle repo from Github,
292-
# without the changes in our current Paddle repo that we
293-
# want to build.
285+
set(${TARGET_NAME}_LIB_PATH "${CMAKE_CURRENT_BINARY_DIR}/${${TARGET_NAME}_LIB_NAME}" CACHE STRING "output library path for target ${TARGET_NAME}")
286+
294287
file(GLOB GO_SOURCE RELATIVE "${CMAKE_CURRENT_SOURCE_DIR}" "*.go")
295288
add_custom_command(TARGET ${TARGET_NAME} POST_BUILD
296-
COMMAND rm "${CMAKE_CURRENT_BINARY_DIR}/${LIB_NAME}"
297-
# Symlink Paddle directory into GOPATH
298-
COMMAND mkdir -p ${PADDLE_IN_GOPATH}
299-
COMMAND rm -rf ${PADDLE_IN_GOPATH}
300-
COMMAND ln -sf ${CMAKE_SOURCE_DIR} ${PADDLE_IN_GOPATH}
301-
# Automatically get all dependencies specified in the source code
302-
COMMAND env GOPATH=${GOPATH} ${CMAKE_Go_COMPILER} get -d ./...
289+
COMMAND rm "${${TARGET_NAME}_LIB_PATH}"
303290
# Golang build source code
304291
COMMAND env GOPATH=${GOPATH} ${CMAKE_Go_COMPILER} build ${BUILD_MODE}
305-
-o "${CMAKE_CURRENT_BINARY_DIR}/${LIB_NAME}"
292+
-o "${${TARGET_NAME}_LIB_PATH}"
306293
${GO_SOURCE}
307294
WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR})
295+
add_dependencies(${TARGET_NAME} go_path)
308296
endfunction(go_library)
309297

310298
function(go_binary TARGET_NAME)

doc/design/build_system/README.md

Lines changed: 45 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -105,3 +105,48 @@ shared_library(api
105105
### Implementation
106106

107107
As above example CMakeLists.txt executes, each function invocation adds "nodes" to a dependency graph. It also use this graph to generate CMake commands including `add_executable`, `add_dependencies`, `target_link_libraries`, and `add_test`.
108+
109+
### Using Package Manager For Go
110+
111+
Building Go binaries and libraries need to satisfy their dependencies, generally
112+
we can do `go get ./...` to download and compile all external dependencies. The
113+
problems are:
114+
115+
1. `go get` will always get the latest code from the default branch of the
116+
remote repo, so changes of dependents might break the build. This is very
117+
different with what we already have in `cmake/external` which download a
118+
specific version or commit id of the dependency.
119+
1. Some locations can not access external dependencies through the internet, as mentioned
120+
in https://github.com/PaddlePaddle/Paddle/issues/2605. Using package management
121+
tools can package the dependencies as a "vendor" package, which can be mirrored
122+
at many cloud file hosting, so users what to compile paddle by themselves can
123+
download this "vendor" package from a mirror site.
124+
125+
#### Choose A Suitable Tool
126+
127+
As mentioned by @wangkuiyi, [Here](https://github.com/golang/go/wiki/PackageManagementTools)
128+
list dozens of Go package managers. We choose the tool using following principles:
129+
130+
- Most "active" projects with more stars, more pull requests or commits
131+
- Widely used project
132+
133+
After comparing all these projects, we shall choose between the most popular
134+
tools: Godep and Glide.
135+
136+
Here's a brief comparison between Godep and Glide
137+
: https://github.com/Masterminds/glide/wiki/Go-Package-Manager-Comparison. There are
138+
also many complaints about using `Godep`. There's also a new "official" pakcage
139+
management tool has been started at: https://github.com/golang/dep to resolve
140+
such problems, but it's currently at Alpha stage. So the best choice now is
141+
glide obviously.
142+
143+
#### Manage Go Packages
144+
145+
- Dependencies: `go/glide.yaml` will store the dependencies and their versions which
146+
is directly imported by paddle. `go/glide.lock` will store all dependencies recursively
147+
with their commit id. Builds will "lock" to these packages if we don't `glide up`
148+
them
149+
- Vendor package: `go/vendor` directory will generated when running `cmake` command. `cmake`
150+
will download the code corresponding to `go/glide.lock`. If we put a vendor folder
151+
under `go/`, cmake will just check the commit id to the packages under the folder,
152+
if commit id matches, there will be no download at all.

0 commit comments

Comments
 (0)