Skip to content

Commit 0433989

Browse files
dan-shawdan-shaw
authored
[tensorflow] add new port for linux (#7568)
* [+] Add tensorflow-cc port (Linux only) * [~] Refactor installation of tensorflow/external 1. Install tensorflow/external to tensorflow-etc/external; 2. Fix TensorflowCCConfig.cmake accordingly. * [tensorflow] Work in progress * fix tensorflow linux config * [tensorflow] partial support for windows * fix config paths * use environmental variables instead * remove files * add python_path * add verbose error messages * review fixes * [tensorflow] refactor config file * minor changes * set arch for CI
1 parent 7f16590 commit 0433989

File tree

5 files changed

+182
-3
lines changed

5 files changed

+182
-3
lines changed

‎ports/tensorflow-cc/CONTROL‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
Source: tensorflow-cc
2+
Version: 1.14
3+
Description: Library for computation using data flow graphs for scalable machine learning
4+
Build-Depends: c-ares

‎ports/tensorflow-cc/TensorflowCCConfig.cmake‎

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
set(tensorflow_cc_INCLUDE_DIR "${CMAKE_CURRENT_LIST_DIR}/../../include")
2+
3+
message(WARNING "Tensorflow has vendored dependencies. You may need to manually include files from tensorflow-external")
4+
set(tensorflow_cc_INCLUDE_DIRS
5+
${tensorflow_cc_INCLUDE_DIR}
6+
${tensorflow_cc_INCLUDE_DIR}/tensorflow-external/
7+
${tensorflow_cc_INCLUDE_DIR}/tensorflow-external/tensorflow/
8+
${tensorflow_cc_INCLUDE_DIR}/tensorflow-external/external/com_google_absl
9+
${tensorflow_cc_INCLUDE_DIR}/tensorflow-external/bazel-out/k8-opt/bin/
10+
${tensorflow_cc_INCLUDE_DIR}/tensorflow-external/external/protobuf_archive/src/
11+
)
12+
13+
add_library(tensorflow_cc::tensorflow_framework SHARED IMPORTED)
14+
set_target_properties(tensorflow_cc::tensorflow_framework
15+
PROPERTIES
16+
IMPORTED_LOCATION ${CMAKE_CURRENT_LIST_DIR}/../../lib/libtensorflow_framework.so.1.14.0
17+
INTERFACE_INCLUDE_DIRECTORIES "${tensorflow_cc_INCLUDE_DIRS}"
18+
)
19+
20+
add_library(tensorflow_cc::tensorflow_cc SHARED IMPORTED)
21+
set_target_properties(tensorflow_cc::tensorflow_cc
22+
PROPERTIES
23+
IMPORTED_LOCATION ${CMAKE_CURRENT_LIST_DIR}/../../lib/libtensorflow_cc.so.1.14.0
24+
INTERFACE_INCLUDE_DIRECTORIES "${tensorflow_cc_INCLUDE_DIRS}"
25+
)
26+
27+
set(tensorflow_cc_FOUND TRUE)
28+
set(tensorflow_framework_FOUND TRUE)

‎ports/tensorflow-cc/file-exists.patch‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
diff --git a/configure.py b/configure.py
2+
index 43af22d..7989b1f 100644
3+
--- a/configure.py
4+
+++ b/configure.py
5+
@@ -183,7 +183,7 @@ def get_python_path(environ_cp, python_bin_path):
6+
7+
paths = []
8+
for path in all_paths:
9+
- if os.path.isdir(path):
10+
+ if os.path.isdir(path) or True:
11+
paths.append(path)
12+
return paths
13+

‎ports/tensorflow-cc/portfile.cmake‎

Lines changed: 116 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,116 @@
1+
include(vcpkg_common_functions)
2+
3+
message(WARNING "This tensorflow port currently is experimental on Windows and Linux platforms.")
4+
5+
vcpkg_from_github(
6+
OUT_SOURCE_PATH SOURCE_PATH
7+
REPO tensorflow/tensorflow
8+
REF v1.14.0
9+
SHA512 ac9ea5a2d1c761aaafbdc335259e29c128127b8d069ec5b206067935180490aa95e93c7e13de57f7f54ce4ba4f34a822face22b4a028f60185edb380e5cd4787
10+
HEAD_REF master
11+
PATCHES
12+
file-exists.patch # required or otherwise it cant find python lib path on windows
13+
)
14+
15+
# due to https://github.com/bazelbuild/bazel/issues/8028, bazel must be version 25.0 or higher
16+
vcpkg_find_acquire_program(BAZEL)
17+
get_filename_component(BAZEL_DIR "${BAZEL}" DIRECTORY)
18+
vcpkg_add_to_path(PREPEND ${BAZEL_DIR})
19+
set(ENV{BAZEL_BIN_PATH} "${BAZEL}")
20+
21+
vcpkg_find_acquire_program(PYTHON3)
22+
get_filename_component(PYTHON3_DIR "${PYTHON3}" DIRECTORY)
23+
vcpkg_add_to_path(PREPEND ${PYTHON3_DIR})
24+
set(ENV{PYTHON_BIN_PATH} "${PYTHON3}")
25+
26+
function(tensorflow_try_remove_recurse_wait PATH_TO_REMOVE)
27+
file(REMOVE_RECURSE ${PATH_TO_REMOVE})
28+
if (EXISTS "${PATH_TO_REMOVE}")
29+
execute_process(COMMAND ${CMAKE_COMMAND} -E sleep 5)
30+
file(REMOVE_RECURSE ${PATH_TO_REMOVE})
31+
endif()
32+
endfunction()
33+
34+
# we currently only support the release version
35+
tensorflow_try_remove_recurse_wait(${CURRENT_BUILDTREES_DIR}/${TARGET_TRIPLET}-rel)
36+
file(MAKE_DIRECTORY ${CURRENT_BUILDTREES_DIR}/${TARGET_TRIPLET}-rel)
37+
file(GLOB SOURCES ${SOURCE_PATH}/*)
38+
file(COPY ${SOURCES} DESTINATION ${CURRENT_BUILDTREES_DIR}/${TARGET_TRIPLET}-rel)
39+
40+
if(CMAKE_HOST_WIN32)
41+
vcpkg_acquire_msys(MSYS_ROOT PACKAGES unzip patch diffutils git)
42+
set(BASH ${MSYS_ROOT}/usr/bin/bash.exe)
43+
set(ENV{BAZEL_SH} ${MSYS_ROOT}/usr/bin/bash.exe)
44+
45+
set(ENV{BAZEL_VS} $ENV{VSInstallDir})
46+
set(ENV{BAZEL_VC} $ENV{VCInstallDir})
47+
endif()
48+
49+
# tensorflow has long file names, which will not work on windows
50+
set(ENV{TEST_TMPDIR} ${CURRENT_BUILDTREES_DIR}/../.bzl)
51+
52+
set(ENV{USE_DEFAULT_PYTHON_LIB_PATH} 1)
53+
set(ENV{TF_NEED_KAFKA} 0)
54+
set(ENV{TF_NEED_OPENCL_SYCL} 0)
55+
set(ENV{TF_NEED_AWS} 0)
56+
set(ENV{TF_NEED_GCP} 0)
57+
set(ENV{TF_NEED_HDFS} 0)
58+
set(ENV{TF_NEED_S3} 0)
59+
set(ENV{TF_ENABLE_XLA} 0)
60+
set(ENV{TF_NEED_GDR} 0)
61+
set(ENV{TF_NEED_VERBS} 0)
62+
set(ENV{TF_NEED_OPENCL} 0)
63+
set(ENV{TF_NEED_MPI} 0)
64+
set(ENV{TF_NEED_TENSORRT} 0)
65+
set(ENV{TF_NEED_NGRAPH} 0)
66+
set(ENV{TF_NEED_IGNITE} 0)
67+
set(ENV{TF_NEED_ROCM} 0)
68+
set(ENV{TF_SET_ANDROID_WORKSPACE} 0)
69+
set(ENV{TF_DOWNLOAD_CLANG} 0)
70+
set(ENV{TF_NCCL_VERSION} 2.3)
71+
set(ENV{NCCL_INSTALL_PATH} "")
72+
set(ENV{CC_OPT_FLAGS} "/arch:AVX")
73+
set(ENV{TF_NEED_CUDA} 0)
74+
75+
message(STATUS "Configuring TensorFlow")
76+
77+
vcpkg_execute_required_process(
78+
COMMAND ${PYTHON3} ${SOURCE_PATH}/configure.py
79+
WORKING_DIRECTORY ${CURRENT_BUILDTREES_DIR}/${TARGET_TRIPLET}-rel
80+
LOGNAME config-${TARGET_TRIPLET}-rel
81+
)
82+
message(STATUS "Warning: Building TensorFlow can take an hour or more.")
83+
84+
if(CMAKE_HOST_WIN32)
85+
vcpkg_execute_build_process(
86+
COMMAND ${BASH} --noprofile --norc -c "${BAZEL} build --verbose_failures -c opt --python_path=${PYTHON3} --incompatible_disable_deprecated_attr_params=false --define=no_tensorflow_py_deps=true ///tensorflow:libtensorflow_cc.so ///tensorflow:install_headers"
87+
WORKING_DIRECTORY ${CURRENT_BUILDTREES_DIR}/${TARGET_TRIPLET}-rel
88+
LOGNAME build-${TARGET_TRIPLET}-rel
89+
)
90+
else()
91+
vcpkg_execute_build_process(
92+
COMMAND ${BAZEL} build --verbose_failures -c opt --python_path=${PYTHON3} --incompatible_disable_deprecated_attr_params=false --define=no_tensorflow_py_deps=true //tensorflow:libtensorflow_cc.so //tensorflow:install_headers
93+
WORKING_DIRECTORY ${CURRENT_BUILDTREES_DIR}/${TARGET_TRIPLET}-rel
94+
LOGNAME build-${TARGET_TRIPLET}-rel
95+
)
96+
endif()
97+
98+
file(COPY ${CURRENT_BUILDTREES_DIR}/${TARGET_TRIPLET}-rel/bazel-genfiles/tensorflow/include/ DESTINATION ${CURRENT_PACKAGES_DIR}/include/tensorflow-external)
99+
100+
if(CMAKE_HOST_WIN32)
101+
file(COPY ${CURRENT_BUILDTREES_DIR}/${TARGET_TRIPLET}-rel/bazel-bin/tensorflow/libtensorflow_cc.so.1.14.0 DESTINATION ${CURRENT_PACKAGES_DIR}/lib)
102+
file(COPY ${CURRENT_BUILDTREES_DIR}/${TARGET_TRIPLET}-rel/bazel-bin/tensorflow/libtensorflow_cc.so.1.14.0.if.lib DESTINATION ${CURRENT_PACKAGES_DIR}/lib)
103+
file(COPY ${CURRENT_BUILDTREES_DIR}/${TARGET_TRIPLET}-rel/bazel-bin/tensorflow/libtensorflow_cc.so.1.14.0 DESTINATION ${CURRENT_PACKAGES_DIR}/debug/lib)
104+
file(COPY ${CURRENT_BUILDTREES_DIR}/${TARGET_TRIPLET}-rel/bazel-bin/tensorflow/libtensorflow_cc.so.1.14.0.if.lib DESTINATION ${CURRENT_PACKAGES_DIR}/debug/lib)
105+
else()
106+
file(COPY ${CURRENT_BUILDTREES_DIR}/${TARGET_TRIPLET}-rel/bazel-bin/tensorflow/libtensorflow_cc.so.1.14.0 DESTINATION ${CURRENT_PACKAGES_DIR}/lib)
107+
file(COPY ${CURRENT_BUILDTREES_DIR}/${TARGET_TRIPLET}-rel/bazel-bin/tensorflow/libtensorflow_framework.so.1.14.0 DESTINATION ${CURRENT_PACKAGES_DIR}/lib)
108+
file(COPY ${CURRENT_BUILDTREES_DIR}/${TARGET_TRIPLET}-rel/bazel-bin/tensorflow/libtensorflow_cc.so.1.14.0 DESTINATION ${CURRENT_PACKAGES_DIR}/debug/lib)
109+
file(COPY ${CURRENT_BUILDTREES_DIR}/${TARGET_TRIPLET}-rel/bazel-bin/tensorflow/libtensorflow_framework.so.1.14.0 DESTINATION ${CURRENT_PACKAGES_DIR}/debug/lib)
110+
endif()
111+
112+
file(COPY ${SOURCE_PATH}/LICENSE DESTINATION ${CURRENT_PACKAGES_DIR}/share/tensorflow-cc)
113+
file(RENAME ${CURRENT_PACKAGES_DIR}/share/tensorflow-cc/LICENSE ${CURRENT_PACKAGES_DIR}/share/tensorflow-cc/copyright)
114+
115+
file(COPY ${CMAKE_CURRENT_LIST_DIR}/TensorflowCCConfig.cmake DESTINATION ${CURRENT_PACKAGES_DIR}/share/unofficial-tensorflow-cc)
116+
file(RENAME ${CURRENT_PACKAGES_DIR}/share/unofficial-tensorflow-cc/TensorflowCCConfig.cmake ${CURRENT_PACKAGES_DIR}/share/unofficial-tensorflow-cc/unofficial-tensorflow-cc-config.cmake)

‎scripts/cmake/vcpkg_find_acquire_program.cmake‎

Lines changed: 21 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -45,6 +45,7 @@ function(vcpkg_find_acquire_program VAR)
4545
unset(_vfa_RENAME)
4646
unset(SUBDIR)
4747
unset(REQUIRED_INTERPRETER)
48+
unset(_vfa_SUPPORTED)
4849
unset(POST_INSTALL_COMMAND)
4950

5051
vcpkg_get_program_files_platform_bitness(PROGRAM_FILES_PLATFORM_BITNESS)
@@ -231,6 +232,23 @@ function(vcpkg_find_acquire_program VAR)
231232
set(URL "http://doxygen.nl/files/doxygen-1.8.15.windows.bin.zip")
232233
set(ARCHIVE "doxygen-1.8.15.windows.bin.zip")
233234
set(HASH 89482dcb1863d381d47812c985593e736d703931d49994e09c7c03ef67e064115d0222b8de1563a7930404c9bc2d3be323f3d13a01ef18861be584db3d5a953c)
235+
elseif(VAR MATCHES "BAZEL")
236+
set(PROGNAME bazel)
237+
set(BAZEL_VERSION 0.25.2)
238+
set(SUBDIR ${BAZEL_VERSION})
239+
set(PATHS ${DOWNLOADS}/tools/bazel/${SUBDIR})
240+
set(_vfa_RENAME "bazel")
241+
if(CMAKE_HOST_SYSTEM_NAME STREQUAL "Linux")
242+
set(_vfa_SUPPORTED ON)
243+
set(URL "https://github.com/bazelbuild/bazel/releases/download/${BAZEL_VERSION}/bazel-${BAZEL_VERSION}-linux-x86_64")
244+
set(ARCHIVE "bazel-${BAZEL_VERSION}-linux-x86_64")
245+
set(NOEXTRACT ON)
246+
set(HASH db4a583cf2996aeb29fd008261b12fe39a4a5faf0fbf96f7124e6d3ffeccf6d9655d391378e68dd0915bc91c9e146a51fd9661963743857ca25179547feceab1)
247+
else()
248+
set(URL "https://github.com/bazelbuild/bazel/releases/download/${BAZEL_VERSION}/bazel-${BAZEL_VERSION}-windows-x86_64.zip")
249+
set(ARCHIVE "bazel-${BAZEL_VERSION}-windows-x86_64.zip")
250+
set(HASH 6482f99a0896f55ef65739e7b53452fd9c0adf597b599d0022a5e0c5fa4374f4a958d46f98e8ba25af4b065adacc578bfedced483d8c169ea5cb1777a99eea53)
251+
endif()
234252
# Download Tools
235253
elseif(VAR MATCHES "ARIA2")
236254
set(PROGNAME aria2c)
@@ -254,7 +272,7 @@ function(vcpkg_find_acquire_program VAR)
254272

255273
do_find()
256274
if("${${VAR}}" MATCHES "-NOTFOUND")
257-
if(NOT CMAKE_HOST_SYSTEM_NAME STREQUAL "Windows")
275+
if(NOT CMAKE_HOST_SYSTEM_NAME STREQUAL "Windows" AND NOT _vfa_SUPPORTED)
258276
set(EXAMPLE ".")
259277
if(DEFINED BREW_PACKAGE_NAME AND CMAKE_HOST_SYSTEM_NAME STREQUAL "Darwin")
260278
set(EXAMPLE ":\n brew install ${BREW_PACKAGE_NAME}")
@@ -274,9 +292,9 @@ function(vcpkg_find_acquire_program VAR)
274292
file(MAKE_DIRECTORY ${PROG_PATH_SUBDIR})
275293
if(DEFINED NOEXTRACT)
276294
if(DEFINED _vfa_RENAME)
277-
file(INSTALL ${ARCHIVE_PATH} DESTINATION ${PROG_PATH_SUBDIR} RENAME ${_vfa_RENAME})
295+
file(INSTALL ${ARCHIVE_PATH} DESTINATION ${PROG_PATH_SUBDIR} RENAME ${_vfa_RENAME} PERMISSIONS OWNER_READ OWNER_WRITE OWNER_EXECUTE GROUP_READ GROUP_EXECUTE WORLD_READ WORLD_EXECUTE)
278296
else()
279-
file(COPY ${ARCHIVE_PATH} DESTINATION ${PROG_PATH_SUBDIR})
297+
file(COPY ${ARCHIVE_PATH} DESTINATION ${PROG_PATH_SUBDIR} PERMISSIONS OWNER_READ OWNER_WRITE OWNER_EXECUTE GROUP_READ GROUP_EXECUTE WORLD_READ WORLD_EXECUTE)
280298
endif()
281299
else()
282300
get_filename_component(ARCHIVE_EXTENSION ${ARCHIVE} EXT)

0 commit comments

Comments
 (0)