ARROW-11212: [Packaging][Python] Use vcpkg as dependency source for manylinux and windows wheels

.env

@@ -18,6 +18,15 @@
 # All of the following environment variables are required to set default values
 # for the parameters in docker-compose.yml.
 
+# empty prefix means that the docker-compose configuration will use named
+# volumes which potentially improves the performance on docker for macos and
+# docker for windows, it also prevents the contamination of the source
+# directory
+# a non-empty prefix means that directories from the host are bind-mounted
+# into the container, it should be set to ".docker/" on github actions to keep
+# the cache plugin functional
+DOCKER_VOLUME_PREFIX=""
+
 ULIMIT_CORE=-1
 REPO=apache/arrow-dev
 ARCH=amd64
@@ -49,3 +58,6 @@ R_IMAGE=ubuntu-gcc-release
 R_TAG=latest
 # -1 does not attempt to install a devtoolset version, any positive integer will install devtoolset-n
 DEVTOOLSET_VERSION=-1
+
+# Used for the manylinux and windows wheels
+VCPKG=c7e96f2a5b73b3278b004aa88abec2f8ebfb43b5

.github/workflows/cpp.yml

@@ -39,6 +39,7 @@ on:
 
 env:
   DOCKER_BUILDKIT: 0
+  DOCKER_VOLUME_PREFIX: ".docker/"
   COMPOSE_DOCKER_CLI_BUILD: 1
   ARROW_ENABLE_TIMING_TESTS: OFF
   ARCHERY_DOCKER_USER: ${{ secrets.DOCKERHUB_USER }}

.github/workflows/cpp_cron.yml

@@ -31,6 +31,7 @@ on:
 
 env:
   DOCKER_BUILDKIT: 0
+  DOCKER_VOLUME_PREFIX: ".docker/"
   COMPOSE_DOCKER_CLI_BUILD: 1
   ARROW_ENABLE_TIMING_TESTS: OFF
   ARCHERY_DOCKER_USER: ${{ secrets.DOCKERHUB_USER }}

.github/workflows/integration.yml

@@ -45,6 +45,7 @@ on:
 
 env:
   DOCKER_BUILDKIT: 0
+  DOCKER_VOLUME_PREFIX: ".docker/"
   COMPOSE_DOCKER_CLI_BUILD: 1
   ARCHERY_DOCKER_USER: ${{ secrets.DOCKERHUB_USER }}
   ARCHERY_DOCKER_PASSWORD: ${{ secrets.DOCKERHUB_TOKEN }}

.github/workflows/java.yml

@@ -37,6 +37,7 @@ on:
 
 env:
   DOCKER_BUILDKIT: 0
+  DOCKER_VOLUME_PREFIX: ".docker/"
   COMPOSE_DOCKER_CLI_BUILD: 1
   ARCHERY_DOCKER_USER: ${{ secrets.DOCKERHUB_USER }}
   ARCHERY_DOCKER_PASSWORD: ${{ secrets.DOCKERHUB_TOKEN }}

.github/workflows/java_jni.yml

@@ -37,6 +37,7 @@ on:
 
 env:
   DOCKER_BUILDKIT: 0
+  DOCKER_VOLUME_PREFIX: ".docker/"
   COMPOSE_DOCKER_CLI_BUILD: 1
   ARCHERY_DOCKER_USER: ${{ secrets.DOCKERHUB_USER }}
   ARCHERY_DOCKER_PASSWORD: ${{ secrets.DOCKERHUB_TOKEN }}

.github/workflows/python.yml

@@ -33,6 +33,7 @@ on:
 
 env:
   DOCKER_BUILDKIT: 0
+  DOCKER_VOLUME_PREFIX: ".docker/"
   COMPOSE_DOCKER_CLI_BUILD: 1
   ARCHERY_DOCKER_USER: ${{ secrets.DOCKERHUB_USER }}
   ARCHERY_DOCKER_PASSWORD: ${{ secrets.DOCKERHUB_TOKEN }}
@@ -50,7 +51,6 @@ jobs:
           - conda-python-3.8-nopandas
           - conda-python-3.6-pandas-0.23
           - conda-python-3.7-pandas-latest
-          - centos-python-3.6-manylinux1
         include:
           - name: conda-python-3.8-nopandas
             cache: conda-python-3.8
@@ -69,10 +69,6 @@ jobs:
             title: AMD64 Conda Python 3.7 Pandas latest
             python: 3.7
             pandas: latest
-          - name: centos-python-3.6-manylinux1
-            cache: manylinux1
-            image: centos-python-manylinux1
-            title: AMD64 CentOS 5.11 Python 3.6 manylinux1
     env:
       PYTHON: ${{ matrix.python || 3.7 }}
       UBUNTU: ${{ matrix.ubuntu || 18.04 }}

.github/workflows/python_cron.yml

@@ -30,6 +30,7 @@ on:
 
 env:
   DOCKER_BUILDKIT: 0
+  DOCKER_VOLUME_PREFIX: ".docker/"
   COMPOSE_DOCKER_CLI_BUILD: 1
   ARCHERY_DOCKER_USER: ${{ secrets.DOCKERHUB_USER }}
   ARCHERY_DOCKER_PASSWORD: ${{ secrets.DOCKERHUB_TOKEN }}

.github/workflows/r.yml

@@ -41,6 +41,7 @@ on:
 
 env:
   DOCKER_BUILDKIT: 0
+  DOCKER_VOLUME_PREFIX: ".docker/"
   COMPOSE_DOCKER_CLI_BUILD: 1
   ARCHERY_DOCKER_USER: ${{ secrets.DOCKERHUB_USER }}
   ARCHERY_DOCKER_PASSWORD: ${{ secrets.DOCKERHUB_TOKEN }}

.github/workflows/ruby.yml

@@ -45,6 +45,7 @@ on:
 
 env:
   DOCKER_BUILDKIT: 0
+  DOCKER_VOLUME_PREFIX: ".docker/"
   COMPOSE_DOCKER_CLI_BUILD: 1
   ARCHERY_DOCKER_USER: ${{ secrets.DOCKERHUB_USER }}
   ARCHERY_DOCKER_PASSWORD: ${{ secrets.DOCKERHUB_TOKEN }}

ci/docker/python-wheel-manylinux-201x.dockerfile

@@ -0,0 +1,103 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+ARG base
+FROM ${base}
+
+RUN yum install -y git flex curl autoconf zip wget
+
+# Install CMake
+ARG cmake=3.19.2
+RUN wget -q https://github.com/Kitware/CMake/releases/download/v${cmake}/cmake-${cmake}-Linux-x86_64.tar.gz -O - | \
+    tar -xzf - --directory /usr/local --strip-components=1
+
+# Install Ninja
+ARG ninja=1.10.2
+RUN mkdir /tmp/ninja && \
+    wget -q https://github.com/ninja-build/ninja/archive/v${ninja}.tar.gz -O - | \
+    tar -xzf - --directory /tmp/ninja --strip-components=1 && \
+    cd /tmp/ninja && \
+    ./configure.py --bootstrap && \
+    mv ninja /usr/local/bin && \
+    rm -rf /tmp/ninja
+
+# Install ccache
+ARG ccache=4.1
+RUN mkdir /tmp/ccache && \
+    wget -q https://github.com/ccache/ccache/archive/v${ccache}.tar.gz -O - | \
+    tar -xzf - --directory /tmp/ccache --strip-components=1 && \
+    cd /tmp/ccache && \
+    mkdir build && \
+    cd build && \
+    cmake -GNinja -DCMAKE_BUILD_TYPE=Release -DZSTD_FROM_INTERNET=ON .. && \
+    ninja install && \
+    rm -rf /tmp/ccache
+
+# Install vcpkg
+ARG vcpkg
+RUN git clone https://github.com/microsoft/vcpkg /opt/vcpkg && \
+    git -C /opt/vcpkg checkout ${vcpkg} && \
+    /opt/vcpkg/bootstrap-vcpkg.sh --useSystemBinaries --disableMetrics && \
+    ln -s /opt/vcpkg/vcpkg /usr/bin/vcpkg
+
+# Patch ports files as needed
+COPY ci/vcpkg arrow/ci/vcpkg
+RUN cd /opt/vcpkg && patch -p1 -i /arrow/ci/vcpkg/ports.patch
+
+ARG build_type=release
+ENV CMAKE_BUILD_TYPE=${build_type} \
+    VCPKG_FORCE_SYSTEM_BINARIES=1 \
+    VCPKG_OVERLAY_TRIPLETS=/arrow/ci/vcpkg \
+    VCPKG_DEFAULT_TRIPLET=x64-linux-static-${build_type}
+
+# TODO(kszucs): factor out the package enumeration to a text file and reuse it
+# from the windows image and potentially in a future macos wheel build
+RUN vcpkg install --clean-after-build \
+        abseil \
+        aws-sdk-cpp[config,cognito-identity,core,identity-management,s3,sts,transfer] \
+        boost-filesystem \
+        boost-regex \
+        brotli \
+        bzip2 \
+        c-ares \
+        curl \
+        flatbuffers \
+        gflags \
+        glog \
+        grpc \
+        lz4 \
+        openssl \
+        orc \
+        protobuf \
+        rapidjson \
+        re2 \
+        snappy \
+        thrift \
+        utf8proc \
+        zlib \
+        zstd
+
+ARG python=3.6
+ENV PYTHON_VERSION=${python}
+RUN PYTHON_ROOT=$(find /opt/python -name cp${PYTHON_VERSION/./}-*) && \
+    echo "export PATH=$PYTHON_ROOT/bin:\$PATH" >> /etc/profile.d/python.sh
+
+SHELL ["/bin/bash", "-i", "-c"]
+ENTRYPOINT ["/bin/bash", "-i", "-c"]
+
+COPY python/requirements-wheel-build.txt /arrow/python/
+RUN pip install -r /arrow/python/requirements-wheel-build.txt

ci/docker/python-wheel-windows-vs2017.dockerfile

@@ -0,0 +1,99 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# based on mcr.microsoft.com/windows/servercore:ltsc2019
+# contains choco and vs2017 preinstalled
+FROM abrarov/msvc-2017:2.10.0
+
+# Install CMake and Ninja
+RUN choco install --no-progress -r -y cmake --installargs 'ADD_CMAKE_TO_PATH=System' && \
+    choco install --no-progress -r -y gzip wget ninja
+
+# Add unix tools to path
+RUN setx path "%path%;C:\Program Files\Git\usr\bin"
+
+# Install vcpkg
+ARG vcpkg
+RUN git clone https://github.com/Microsoft/vcpkg && \
+    git -C vcpkg checkout %vcpkg% && \
+    vcpkg\bootstrap-vcpkg.bat -disableMetrics -win64 && \
+    setx PATH "%PATH%;C:\vcpkg"
+
+# Patch ports files as needed
+COPY ci/vcpkg arrow/ci/vcpkg
+RUN cd vcpkg && patch -p1 -i C:/arrow/ci/vcpkg/ports.patch
+
+# Configure vcpkg and install dependencies
+# NOTE: use windows batch environment notation for build arguments in RUN
+# statements but bash notation in ENV statements
+# VCPKG_FORCE_SYSTEM_BINARIES=1 spare around ~750MB of image size if the system
+# cmake's and ninja's versions are recent enough
+COPY ci/vcpkg arrow/ci/vcpkg
+ARG build_type=release
+ENV CMAKE_BUILD_TYPE=${build_type} \
+    VCPKG_OVERLAY_TRIPLETS=C:\\arrow\\ci\\vcpkg \
+    VCPKG_DEFAULT_TRIPLET=x64-windows-static-md-${build_type}
+RUN vcpkg install --clean-after-build \
+        abseil \
+        aws-sdk-cpp[config,cognito-identity,core,identity-management,s3,sts,transfer] \
+        boost-filesystem \
+        boost-multiprecision \
+        boost-regex \
+        boost-system \
+        brotli \
+        bzip2 \
+        c-ares \
+        curl \
+        flatbuffers \
+        gflags \
+        glog \
+        grpc \
+        lz4 \
+        openssl \
+        orc \
+        protobuf \
+        rapidjson \
+        re2 \
+        snappy \
+        thrift \
+        utf8proc \
+        zlib \
+        zstd
+
+# Remove previous installations of python from the base image
+RUN wmic product where "name like 'python%%'" call uninstall /nointeractive && \
+    rm -rf Python*
+
+# Define the full version number otherwise choco falls back to patch number 0 (3.7 => 3.7.0)
+ARG python=3.6
+RUN (if "%python%"=="3.6" setx PYTHON_VERSION 3.6.8) & \
+    (if "%python%"=="3.7" setx PYTHON_VERSION 3.7.4) & \
+    (if "%python%"=="3.8" setx PYTHON_VERSION 3.8.6) & \
+    (if "%python%"=="3.9" setx PYTHON_VERSION 3.9.1)
+RUN choco install -r -y --no-progress python --version=%PYTHON_VERSION%
+RUN python -m pip install -U pip
+
+COPY python/requirements-wheel-build.txt arrow/python/
+RUN pip install -r arrow/python/requirements-wheel-build.txt
+
+# TODO(kszucs): set clcache as the compiler
+ENV CLCACHE_DIR="C:\clcache"
+RUN pip install clcache
+
+# For debugging purposes
+# RUN wget --no-check-certificate https://github.com/lucasg/Dependencies/releases/download/v1.10/Dependencies_x64_Release.zip
+# RUN unzip Dependencies_x64_Release.zip -d Dependencies && setx path "%path%;C:\Depencencies"