ARROW-16102: [C++] Add support for building with system gRPC and bundled GCS

.github/workflows/cpp.yml

@@ -126,6 +126,7 @@ jobs:
       ARROW_DATASET: ON
       ARROW_FLIGHT: ON
       ARROW_GANDIVA: ON
+      ARROW_GCS: ON
       ARROW_HDFS: ON
       ARROW_HOME: /usr/local
       ARROW_JEMALLOC: ON
@@ -141,6 +142,8 @@ jobs:
       ARROW_WITH_SNAPPY: ON
       ARROW_WITH_ZLIB: ON
       ARROW_WITH_ZSTD: ON
+      # System Abseil installed by Homebrew uses C++ 17
+      CMAKE_CXX_STANDARD: 17
     steps:
       - name: Checkout Arrow
         uses: actions/checkout@v2
@@ -153,6 +156,9 @@ jobs:
           rm -f /usr/local/bin/2to3
           brew update --preinstall
           brew bundle --file=cpp/Brewfile
+      - name: Install Google Cloud Storage Testbench
+        shell: bash
+        run: ci/scripts/install_gcs_testbench.sh default
       - name: Setup ccache
         run: |
           ci/scripts/ccache_setup.sh
@@ -268,6 +274,9 @@ jobs:
       ARROW_DATASET: ON
       ARROW_FLIGHT: ON
       ARROW_GANDIVA: ON
+      # google-could-cpp uses _dupenv_s() but it can't be used with msvcrt.
+      # We need to use ucrt to use _dupenv_s().
+      # ARROW_GCS: ON
       ARROW_HDFS: OFF
       ARROW_HOME: /mingw${{ matrix.mingw-n-bits }}
       ARROW_JEMALLOC: OFF

.github/workflows/ruby.yml

@@ -90,6 +90,7 @@ jobs:
           ulimit -c unlimited
           archery docker run \
             -e ARROW_FLIGHT=ON \
+            -e ARROW_GCS=ON \
             -e Protobuf_SOURCE=BUNDLED \
             -e gRPC_SOURCE=BUNDLED \
             ubuntu-ruby
@@ -110,6 +111,7 @@ jobs:
       ARROW_BUILD_TESTS: OFF
       ARROW_FLIGHT: ON
       ARROW_GANDIVA: ON
+      ARROW_GCS: ON
       ARROW_GLIB_GTK_DOC: true
       ARROW_GLIB_WERROR: true
       ARROW_HOME: /usr/local
@@ -188,6 +190,7 @@ jobs:
       ARROW_BUILD_TYPE: release
       ARROW_FLIGHT: ON
       ARROW_GANDIVA: ON
+      ARROW_GCS: ON
       ARROW_HDFS: OFF
       ARROW_HOME: /ucrt${{ matrix.mingw-n-bits }}
       ARROW_JEMALLOC: OFF

ci/scripts/install_gcs_testbench.sh

@@ -31,7 +31,8 @@ fi
 
 version=$1
 if [[ "${version}" -eq "default" ]]; then
-  version="v0.7.0"
+  version="v0.16.0"
 fi
 
-pip install "https://github.com/googleapis/storage-testbench/archive/${version}.tar.gz"
+${PYTHON:-python3} -m pip install \
+  "https://github.com/googleapis/storage-testbench/archive/${version}.tar.gz"

cpp/cmake_modules/ThirdpartyToolchain.cmake

@@ -2593,8 +2593,9 @@ endmacro()
 # ----------------------------------------------------------------------
 # Dependencies for Arrow Flight RPC
 
-macro(build_absl_once)
-  if(NOT TARGET absl_ep)
+macro(resolve_dependency_absl)
+  # Choose one of built absl::* targets
+  if(NOT TARGET absl::algorithm)
     message(STATUS "Building Abseil-cpp from source")
     set(ABSL_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/absl_ep-install")
     set(ABSL_INCLUDE_DIR "${ABSL_PREFIX}/include")
@@ -3451,6 +3452,7 @@ macro(build_absl_once)
     # Work around https://gitlab.kitware.com/cmake/cmake/issues/15052
     file(MAKE_DIRECTORY ${ABSL_INCLUDE_DIR})
 
+    set(ABSL_VENDORED TRUE)
   endif()
 endmacro()
 
@@ -3464,8 +3466,8 @@ macro(build_grpc)
   get_target_property(c-ares_INCLUDE_DIR c-ares::cares INTERFACE_INCLUDE_DIRECTORIES)
   include_directories(SYSTEM ${c-ares_INCLUDE_DIR})
 
-  # First need to build Abseil
-  build_absl_once()
+  # First need Abseil
+  resolve_dependency_absl()
 
   message(STATUS "Building gRPC from source")
 
@@ -3498,7 +3500,9 @@ macro(build_grpc)
 
   add_custom_target(grpc_dependencies)
 
-  add_dependencies(grpc_dependencies absl_ep)
+  if(ABSL_VENDORED)
+    add_dependencies(grpc_dependencies absl_ep)
+  endif()
   if(CARES_VENDORED)
     add_dependencies(grpc_dependencies cares_ep)
   endif()
@@ -3819,7 +3823,7 @@ macro(build_google_cloud_cpp_storage)
   message(STATUS "Only building the google-cloud-cpp::storage component")
 
   # List of dependencies taken from https://github.com/googleapis/google-cloud-cpp/blob/master/doc/packaging.md
-  build_absl_once()
+  resolve_dependency_absl()
   build_crc32c_once()
 
   # Curl is required on all platforms, but building it internally might also trip over S3's copy.
@@ -3830,7 +3834,9 @@ macro(build_google_cloud_cpp_storage)
   # Build google-cloud-cpp, with only storage_client
 
   # Inject vendored packages via CMAKE_PREFIX_PATH
-  list(APPEND GOOGLE_CLOUD_CPP_PREFIX_PATH_LIST ${ABSL_PREFIX})
+  if(ABSL_VENDORED)
+    list(APPEND GOOGLE_CLOUD_CPP_PREFIX_PATH_LIST ${ABSL_PREFIX})
+  endif()
   list(APPEND GOOGLE_CLOUD_CPP_PREFIX_PATH_LIST ${CRC32C_PREFIX})
   list(APPEND GOOGLE_CLOUD_CPP_PREFIX_PATH_LIST ${NLOHMANN_JSON_PREFIX})
 
@@ -3862,7 +3868,9 @@ macro(build_google_cloud_cpp_storage)
 
   add_custom_target(google_cloud_cpp_dependencies)
 
-  add_dependencies(google_cloud_cpp_dependencies absl_ep)
+  if(ABSL_VENDORED)
+    add_dependencies(google_cloud_cpp_dependencies absl_ep)
+  endif()
   add_dependencies(google_cloud_cpp_dependencies crc32c_ep)
   add_dependencies(google_cloud_cpp_dependencies nlohmann_json::nlohmann_json)
 
@@ -3903,7 +3911,8 @@ macro(build_google_cloud_cpp_storage)
                         absl::memory
                         absl::optional
                         absl::time
-                        Threads::Threads)
+                        Threads::Threads
+                        OpenSSL::Crypto)
 
   add_library(google-cloud-cpp::storage STATIC IMPORTED)
   set_target_properties(google-cloud-cpp::storage

cpp/thirdparty/versions.txt

@@ -49,8 +49,8 @@ ARROW_GFLAGS_BUILD_VERSION=v2.2.2
 ARROW_GFLAGS_BUILD_SHA256_CHECKSUM=34af2f15cf7367513b352bdcd2493ab14ce43692d2dcd9dfc499492966c64dcf
 ARROW_GLOG_BUILD_VERSION=v0.5.0
 ARROW_GLOG_BUILD_SHA256_CHECKSUM=eede71f28371bf39aa69b45de23b329d37214016e2055269b3b5e7cfd40b59f5
-ARROW_GOOGLE_CLOUD_CPP_BUILD_VERSION=v1.35.0
-ARROW_GOOGLE_CLOUD_CPP_BUILD_SHA256_CHECKSUM=e4e9eac1e7999eff195db270bc2a719004660b3730ebb5d2f444f2d2057e49b2
+ARROW_GOOGLE_CLOUD_CPP_BUILD_VERSION=v1.39.0
+ARROW_GOOGLE_CLOUD_CPP_BUILD_SHA256_CHECKSUM=73e4e840018b24bec2beb49e036a3c2d8c471d4dc4a18b9026ccc4d8ab8e78cc
 ARROW_GRPC_BUILD_VERSION=v1.35.0
 ARROW_GRPC_BUILD_SHA256_CHECKSUM=27dd2fc5c9809ddcde8eb6fa1fa278a3486566dfc28335fca13eb8df8bd3b958
 ARROW_GTEST_BUILD_VERSION=1.11.0
@@ -119,7 +119,7 @@ DEPENDENCIES=(
   "ARROW_GBENCHMARK_URL gbenchmark-${ARROW_GBENCHMARK_BUILD_VERSION}.tar.gz https://github.com/google/benchmark/archive/${ARROW_GBENCHMARK_BUILD_VERSION}.tar.gz"
   "ARROW_GFLAGS_URL gflags-${ARROW_GFLAGS_BUILD_VERSION}.tar.gz https://github.com/gflags/gflags/archive/${ARROW_GFLAGS_BUILD_VERSION}.tar.gz"
   "ARROW_GLOG_URL glog-${ARROW_GLOG_BUILD_VERSION}.tar.gz https://github.com/google/glog/archive/${ARROW_GLOG_BUILD_VERSION}.tar.gz"
-  "ARROW_GOOGLE_CLOUD_CPP_URL google-cloud-cpp-${ARROW_GOOGLE_CLOUD_CPP_BUILD_VERSION}.tar.gz https://github.com/googleapis/google-cloud-cpp/archive/refs/tags/${ARROW_GOOGLE_CLOUD_CPP_BUILD_VERSION}.tar.gz"
+  "ARROW_GOOGLE_CLOUD_CPP_URL google-cloud-cpp-${ARROW_GOOGLE_CLOUD_CPP_BUILD_VERSION}.tar.gz https://github.com/googleapis/google-cloud-cpp/archive/${ARROW_GOOGLE_CLOUD_CPP_BUILD_VERSION}.tar.gz"
   "ARROW_GRPC_URL grpc-${ARROW_GRPC_BUILD_VERSION}.tar.gz https://github.com/grpc/grpc/archive/${ARROW_GRPC_BUILD_VERSION}.tar.gz"
   "ARROW_GTEST_URL gtest-${ARROW_GTEST_BUILD_VERSION}.tar.gz https://github.com/google/googletest/archive/release-${ARROW_GTEST_BUILD_VERSION}.tar.gz"
   "ARROW_JEMALLOC_URL jemalloc-${ARROW_JEMALLOC_BUILD_VERSION}.tar.bz2 https://github.com/jemalloc/jemalloc/releases/download/${ARROW_JEMALLOC_BUILD_VERSION}/jemalloc-${ARROW_JEMALLOC_BUILD_VERSION}.tar.bz2"