apache · terencehonles · Dec 11, 2020
diff --git a/dev/release/verify-release-candidate.sh b/dev/release/verify-release-candidate.sh
@@ -656,7 +656,7 @@ test_linux_wheels() {
 }
 
 test_macos_wheels() {
-  local py_arches="3.5m 3.6m 3.7m 3.8"
+  local py_arches="3.5m 3.6m 3.7m 3.8 3.9"
 
   for py_arch in ${py_arches}; do
     local env=_verify_wheel-${py_arch}

diff --git a/dev/tasks/python-wheels/github.osx.yml b/dev/tasks/python-wheels/github.osx.yml
@@ -0,0 +1,142 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# NOTE: must set "Crossbow" as name to have the badge links working in the
+# github comment reports!
+name: Crossbow
+
+on:
+  push:
+    branches:
+      - "*-github-*"
+
+jobs:
+  build:
+    name: Build wheel for OS X
+    runs-on: macos-latest
+    steps:
+      - name: Checkout Arrow
+        run: |
+          set -ex
+
+          git clone --no-checkout {{ arrow.remote }} arrow
+          git -C arrow fetch -t {{ arrow.remote }} {{ arrow.branch }}
+          git -C arrow checkout FETCH_HEAD
+          git -C arrow submodule update --init --recursive
+
+      - name: Build wheel
+        run: |
+          set -ex
+
+          git clone https://github.com/matthew-brett/multibuild
+          git -C multibuild checkout 8882150df6529658700b66bec124dfb77eefca26
+
+          # source utilities required for wheel builds
+          export CONFIG_PATH=`pwd`/arrow/dev/tasks/python-wheels/osx-build.sh
+          source multibuild/common_utils.sh
+          source multibuild/travis_osx_steps.sh
+
+          # provided by travis_osx_steps
+          before_install
+
+          brew install \
+            automake \
+            bison \
+            boost \
+            ccache \
+            cmake \
+            flatbuffers \
+            flex \
+            git \
+            openblas \
+            [email protected] \
+            protobuf \
+            [email protected] \
+            thrift \
+            wget
+
+          # overridden by osx-build.sh
+          build_wheel arrow
+
+          # test the built wheels, move Homebrew directory to ensure things
+          # are properly statically-linked
+          CELLAR=$(brew --cellar)
+          sudo mv $CELLAR{,.bak}
+
+          # make a new Cellar, allowing numpy to dynamically link to openblas
+          sudo mkdir $CELLAR
+          sudo ln -s ../Cellar.bak/openblas $CELLAR/openblas
+          # and brew's openblas will need gcc
+          sudo ln -s ../Cellar.bak/gcc $CELLAR/gcc
+
+          # install the built wheel and test dependencies (osx-build.sh)
+          install_wheel arrow
+
+          # run unit tests before removing the system libraries (osx-build.sh)
+          ulimit -S -n 4096 && run_unit_tests arrow
+
+          # run the import tests (osx-build.sh)
+          run_import_tests
+
+          # restore the original Cellar
+          sudo rm $CELLAR/{gcc,openblas}
+          sudo rmdir $CELLAR
+          sudo mv $CELLAR{.bak,}
+        env:
+          # pyarrow config
+          ARROW_S3: {{ arrow_s3 }}
+          MACOSX_DEPLOYMENT_TARGET: {{ macos_deployment_target }}
+          PYARROW_BUILD_VERBOSE: 1
+          PYARROW_VERSION: {{ arrow.no_rc_version }}
+
+          # multibuild config
+          MB_PYTHON_VERSION: {{ python_version }}
+          PLAT: x86_64
+
+      - name: Setup Crossbow
+        run: |
+          pip3 install \
+            click \
+            github3.py \
+            jinja2 \
+            jira \
+            pygit2 \
+            ruamel.yaml \
+            setuptools_scm \
+            toolz
+
+      - name: Upload artifacts
+        run: |
+          python3 arrow/dev/tasks/crossbow.py \
+            --queue-path $(pwd) \
+            --queue-remote {{ queue_remote_url }} \
+            upload-artifacts \
+            --sha {{ task.branch }} \
+            --tag {{ task.tag }} \
+            --pattern "arrow/python/dist/*.whl"
+        env:
+          CROSSBOW_GITHUB_TOKEN: {{ '${{ secrets.CROSSBOW_GITHUB_TOKEN }}' }}
+
+      {% if arrow.branch == 'master' %}
+      - name: Upload package to Gemfury
+        run: |
+          # upload to gemfury pypi repository, there should be a single wheel
+          path=$(ls arrow/python/dist/*.whl)
+          curl -F "package=@${path}" https://${CROSSBOW_GEMFURY_TOKEN}@push.fury.io/${CROSSBOW_GEMFURY_ORG}/
+        env:
+          CROSSBOW_GEMFURY_TOKEN: {{ '${{ secrets.CROSSBOW_GEMFURY_TOKEN }}' }}
+          CROSSBOW_GEMFURY_ORG: {{ '${{ secrets.CROSSBOW_GEMFURY_ORG }}' }}
+      {% endif %}
diff --git a/dev/tasks/python-wheels/osx-build.sh b/dev/tasks/python-wheels/osx-build.sh
@@ -50,7 +50,10 @@ function build_wheel {
     export ARROW_HOME=`pwd`/arrow-dist
     export PARQUET_HOME=`pwd`/arrow-dist
 
-    pip install $(pip_opts) -r python/requirements-wheel-build.txt
+    # If NumPy builds from the source make sure it is built against OpenBLAS
+    # See: https://github.com/numpy/numpy/issues/15947#issuecomment-686159427
+    OPENBLAS=$(brew --prefix openblas) pip install \
+        $(pip_opts) -r python/requirements-wheel-build.txt
 
     git submodule update --init
     export ARROW_TEST_DATA=`pwd`/testing/data

diff --git a/dev/tasks/python-wheels/travis.osx.yml b/dev/tasks/python-wheels/travis.osx.yml
diff --git a/dev/tasks/tasks.yml b/dev/tasks/tasks.yml
@@ -290,6 +290,15 @@ tasks:
       - arrow-cpp-{no_rc_version}-py38(h[a-z0-9]+)_0_cpu.tar.bz2
       - pyarrow-{no_rc_version}-py38(h[a-z0-9]+)_0_cpu.tar.bz2
 
+  conda-osx-clang-py39:
+    ci: azure
+    template: conda-recipes/azure.osx.yml
+    params:
+      config: osx_python3.9.____cpython
+    artifacts:
+      - arrow-cpp-{no_rc_version}-py39(h[a-z0-9]+)_0_cpu.tar.bz2
+      - pyarrow-{no_rc_version}-py39(h[a-z0-9]+)_0_cpu.tar.bz2
+
   ############################## Conda Windows ################################
 
   conda-win-vs2017-py36-r36:
@@ -494,8 +503,8 @@ tasks:
   ############################## Wheel OSX ####################################
 
   wheel-osx-mavericks-cp36m:
-    ci: travis
-    template: python-wheels/travis.osx.yml
+    ci: github
+    template: python-wheels/github.osx.yml
     params:
       python_version: 3.6
       macos_deployment_target: 10.9
@@ -504,8 +513,8 @@ tasks:
       - pyarrow-{no_rc_version}-cp36-cp36m-macosx_10_9_x86_64.whl
 
   wheel-osx-mavericks-cp37m:
-    ci: travis
-    template: python-wheels/travis.osx.yml
+    ci: github
+    template: python-wheels/github.osx.yml
     params:
       python_version: 3.7
       macos_deployment_target: 10.9
@@ -514,20 +523,30 @@ tasks:
       - pyarrow-{no_rc_version}-cp37-cp37m-macosx_10_9_x86_64.whl
 
   wheel-osx-mavericks-cp38:
-    ci: travis
-    template: python-wheels/travis.osx.yml
+    ci: github
+    template: python-wheels/github.osx.yml
     params:
       python_version: 3.8
       macos_deployment_target: 10.9
       arrow_s3: "OFF"
     artifacts:
       - pyarrow-{no_rc_version}-cp38-cp38-macosx_10_9_x86_64.whl
 
+  wheel-osx-mavericks-cp39:
+    ci: github
+    template: python-wheels/github.osx.yml
+    params:
+      python_version: 3.9
+      macos_deployment_target: 10.9
+      arrow_s3: "OFF"
+    artifacts:
+      - pyarrow-{no_rc_version}-cp39-cp39-macosx_10_9_x86_64.whl
+
   # enable S3 support from macOS 10.13 so we don't need to bundle curl, crypt and ssl
 
   wheel-osx-high-sierra-cp36m:
-    ci: travis
-    template: python-wheels/travis.osx.yml
+    ci: github
+    template: python-wheels/github.osx.yml
     params:
       python_version: 3.6
       macos_deployment_target: 10.13
@@ -536,8 +555,8 @@ tasks:
       - pyarrow-{no_rc_version}-cp36-cp36m-macosx_10_13_x86_64.whl
 
   wheel-osx-high-sierra-cp37m:
-    ci: travis
-    template: python-wheels/travis.osx.yml
+    ci: github
+    template: python-wheels/github.osx.yml
     params:
       python_version: 3.7
       macos_deployment_target: 10.13
@@ -546,15 +565,25 @@ tasks:
       - pyarrow-{no_rc_version}-cp37-cp37m-macosx_10_13_x86_64.whl
 
   wheel-osx-high-sierra-cp38:
-    ci: travis
-    template: python-wheels/travis.osx.yml
+    ci: github
+    template: python-wheels/github.osx.yml
     params:
       python_version: 3.8
       macos_deployment_target: 10.13
       arrow_s3: "ON"
     artifacts:
       - pyarrow-{no_rc_version}-cp38-cp38-macosx_10_13_x86_64.whl
 
+  wheel-osx-high-sierra-cp39:
+    ci: github
+    template: python-wheels/github.osx.yml
+    params:
+      python_version: 3.9
+      macos_deployment_target: 10.13
+      arrow_s3: "ON"
+    artifacts:
+      - pyarrow-{no_rc_version}-cp39-cp39-macosx_10_13_x86_64.whl
+
   ############################## Wheel Windows ################################
 
   wheel-win-cp36m: