aimrocks packaging pipeline #214
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: aimrocks packaging pipeline | |
| on: workflow_dispatch | |
| jobs: | |
| uploading-wheels: | |
| needs: | |
| - linux-dist-aarch64 | |
| - linux-dist-x86_64 | |
| - macos-dist | |
| runs-on: ubuntu-22.04 | |
| name: Uploading wheels | |
| steps: | |
| - name: Setup python | |
| uses: actions/setup-python@v2 | |
| with: | |
| python-version: '3.9' | |
| architecture: x64 | |
| - name: Install dev dependencies | |
| run: | | |
| python -m pip install twine | |
| - name: Download all the wheels | |
| uses: actions/download-artifact@v4 | |
| with: | |
| merge-multiple: true | |
| path: artifact | |
| - name: Display structure of downloaded files | |
| run: ls -R | |
| - name: Publish wheels | |
| env: | |
| PYPI_PASSWORD: ${{ secrets.pypi_password }} | |
| run: | | |
| python -m twine upload -u __token__ -p "${PYPI_PASSWORD}" artifact/*.whl | |
| linux-dist-aarch64: | |
| runs-on: m1 | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| manylinux-version: | |
| - {image: 'manylinux_2_28_aarch64', tag: '2024-04-23-ef7507e'} | |
| - {image: 'manylinux2014_aarch64', tag: '2024-03-25-9206bd9'} | |
| - {image: 'manylinux_2_24_aarch64', tag: 'latest'} | |
| name: ${{ matrix.manylinux-version.image }} using Docker on M1 Mac | |
| steps: | |
| - name: Checkout sources | |
| uses: actions/checkout@v5 | |
| - name: Pull Docker Images | |
| run: | | |
| docker pull quay.io/pypa/${{ matrix.manylinux-version.image }}:${{ matrix.manylinux-version.tag }} | |
| - name: Building dependencies for ${{ matrix.manylinux-version.image }} | |
| run: | | |
| docker build --no-cache \ | |
| --build-arg FROM=quay.io/pypa/${{ matrix.manylinux-version.image }}:${{ matrix.manylinux-version.tag }} \ | |
| --target deps . | |
| - name: Building rocksdb for ${{ matrix.manylinux-version.image }} | |
| run: | | |
| docker build --no-cache \ | |
| --build-arg FROM=quay.io/pypa/${{ matrix.manylinux-version.image }}:${{ matrix.manylinux-version.tag }} \ | |
| --target rocksdb . | |
| - name: Building wheels for ${{ matrix.manylinux-version.image }} | |
| run: | | |
| docker build --no-cache \ | |
| --build-arg FROM=quay.io/pypa/${{ matrix.manylinux-version.image }}:${{ matrix.manylinux-version.tag }} \ | |
| --target wheels . \ | |
| -t aimhubio/aimrocks:${{ matrix.manylinux-version.image }} | |
| - name: Auditing wheels for ${{ matrix.manylinux-version.image }} | |
| run: | | |
| mkdir -p manylinux_dist/ && \ | |
| docker run --rm \ | |
| --mount type=bind,source=$PWD/manylinux_dist,target=/opt/aimrocks/manylinux_dist \ | |
| aimhubio/aimrocks:${{ matrix.manylinux-version.image }} \ | |
| bash -e /opt/aimrocks/docker/audit-wheels.sh | |
| - uses: actions/upload-artifact@v4 | |
| with: | |
| path: manylinux_dist/*.whl | |
| name: artifact-${{ matrix.manylinux-version.image }} | |
| if-no-files-found: error | |
| linux-dist-x86_64: | |
| runs-on: ubuntu-22.04 | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| manylinux-version: | |
| - {image: 'manylinux1_x86_64', tag: 'latest'} | |
| - {image: 'manylinux2010_x86_64', tag: 'latest'} | |
| - {image: 'manylinux2014_x86_64', tag: '2024-04-23-ef7507e'} | |
| - {image: 'manylinux_2_24_x86_64', tag: 'latest'} | |
| name: ${{ matrix.manylinux-version.image }} | |
| steps: | |
| - name: Install Docker & images | |
| run: | | |
| apt update && apt install -y docker.io | |
| sudo systemctl enable --now docker | |
| - name: Checkout sources | |
| uses: actions/checkout@v5 | |
| - name: Setup python | |
| uses: actions/setup-python@v2 | |
| with: | |
| python-version: '3.9' | |
| architecture: x64 | |
| - name: Pull Docker Images | |
| run: | | |
| docker pull quay.io/pypa/${{ matrix.manylinux-version.image }}:${{ matrix.manylinux-version.tag }} | |
| - uses: satackey/[email protected] | |
| # Ignore the failure of a step and avoid terminating the job. | |
| continue-on-error: true | |
| with: | |
| key: aimrocks-cython-manylinux-build-${{ matrix.manylinux-version.image }}-{hash} | |
| restore-keys: | | |
| aimrocks-cython-manylinux-build-${{ matrix.manylinux-version.image }}- | |
| - name: Building dependencies for ${{ matrix.manylinux-version.image }} | |
| run: | | |
| docker build --no-cache \ | |
| --build-arg FROM=quay.io/pypa/${{ matrix.manylinux-version.image }}:${{ matrix.manylinux-version.tag }} \ | |
| --target deps . | |
| - name: Building rocksdb for ${{ matrix.manylinux-version.image }} | |
| run: | | |
| docker build --no-cache \ | |
| --build-arg FROM=quay.io/pypa/${{ matrix.manylinux-version.image }}:${{ matrix.manylinux-version.tag }} \ | |
| --target rocksdb . | |
| - name: Building wheels for ${{ matrix.manylinux-version.image }} | |
| run: | | |
| docker build --no-cache \ | |
| --build-arg FROM=quay.io/pypa/${{ matrix.manylinux-version.image }}:${{ matrix.manylinux-version.tag }} \ | |
| --target wheels . \ | |
| -t aimhubio/aimrocks:${{ matrix.manylinux-version.image }} | |
| - name: Auditing wheels for ${{ matrix.manylinux-version.image }} | |
| run: | | |
| mkdir -p manylinux_dist/ && \ | |
| docker run --rm \ | |
| --mount type=bind,source=$PWD/manylinux_dist,target=/opt/aimrocks/manylinux_dist \ | |
| aimhubio/aimrocks:${{ matrix.manylinux-version.image }} \ | |
| bash -e /opt/aimrocks/docker/audit-wheels.sh | |
| - uses: actions/upload-artifact@v4 | |
| with: | |
| path: manylinux_dist/*.whl | |
| name: artifact-${{ matrix.manylinux-version.image }} | |
| if-no-files-found: error | |
| macos-deps: | |
| runs-on: m1 | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| arch: ['arm64', 'x86_64'] | |
| name: Preparing dependencies for ${{ matrix.arch }} Mac build | |
| env: | |
| MACOSX_DEPLOYMENT_TARGET: 10.14 | |
| AIM_DEP_DIR: /tmp/run/${{ github.run_number }}/${{ matrix.arch }} | |
| CMAKE: /opt/homebrew/bin/cmake | |
| steps: | |
| - name: Install tmate | |
| if: matrix.arch == 'arm64' | |
| run: | | |
| mkdir -p /tmp/env | |
| export CONDA_PKGS_DIRS=/tmp/.conda/pkgs | |
| mkdir -p /tmp/.conda/pkgs | |
| /opt/conda/${{ matrix.arch }}/bin/conda create -y --prefix /tmp/env python=3.11 tmate -c conda-forge | |
| - name: Setup tmate session 1 | |
| if: matrix.arch == 'arm64' | |
| run: | | |
| unset TMUX | |
| /tmp/env/bin/tmate -S /tmp/tmate.sock new-session -d | |
| - name: Setup tmate session 2 | |
| if: matrix.arch == 'arm64' | |
| run: | | |
| /tmp/env/bin/tmate -S /tmp/tmate.sock wait tmate-ready | |
| - name: Setup tmate session 3 | |
| if: matrix.arch == 'arm64' | |
| run: | | |
| echo "SSH: $(/tmp/env/bin/tmate -S /tmp/tmate.sock display -p '#{tmate_ssh}')" | tee /tmp/tmate.txt | |
| echo "WEB: $(/tmp/env/bin/tmate -S /tmp/tmate.sock display -p '#{tmate_web}')" | tee -a /tmp/tmate.txt | |
| # Keep job alive for remote debugging | |
| - name: Setup tmate session 4 | |
| if: matrix.arch == 'arm64' | |
| run: | | |
| tail -f /dev/null | |
| timeout-minutes: 15 | |
| - name: Preparing Build Dir for Dependencies | |
| run: | | |
| mkdir -p $AIM_DEP_DIR | |
| rm -rf $AIM_DEP_DIR/* | |
| - name: Checkout sources | |
| uses: actions/checkout@v5 | |
| - name: Building ZLib | |
| run: | | |
| pwd | |
| ls -lhatr .github/workflows/mac | |
| arch -${{matrix.arch}} ./.github/workflows/mac/build-zlib.sh | |
| - name: Building BZip2 | |
| run: | | |
| arch -${{matrix.arch}} ./.github/workflows/mac/build-bzip2.sh | |
| - name: Building LZ4 | |
| run: | | |
| arch -${{matrix.arch}} ./.github/workflows/mac/build-lz4.sh | |
| - name: Building Snappy | |
| run: | | |
| arch -${{matrix.arch}} ./.github/workflows/mac/build-snappy.sh | |
| - name: Building ZSTD | |
| run: | | |
| arch -${{matrix.arch}} ./.github/workflows/mac/build-zstd.sh | |
| - name: Building RocksDB | |
| run: | | |
| arch -${{matrix.arch}} ./.github/workflows/mac/build-rocksdb.sh | |
| macos-dist: | |
| runs-on: m1 | |
| needs: macos-deps | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| python-version: [ '3.6', '3.7', '3.8', '3.9', '3.10' , '3.11', '3.12', '3.13'] | |
| arch: ['arm64', 'x86_64'] | |
| exclude: | |
| - arch: 'arm64' | |
| python-version: 3.6 | |
| - arch: 'arm64' | |
| python-version: 3.7 | |
| # requirement for custom runners: having cmake and python installed | |
| name: Python ${{ matrix.python-version }} for ${{ matrix.arch }} build | |
| env: | |
| MACOSX_DEPLOYMENT_TARGET: 10.14 | |
| AIM_DEP_DIR: /tmp/run/${{ github.run_number }}/${{ matrix.arch }} | |
| PYTHON: /opt/conda/${{ matrix.arch }}/envs/py${{ matrix.python-version }}/bin/python | |
| steps: | |
| - name: Build and test wheels | |
| run: | | |
| arch -${{matrix.arch}} $PYTHON -m build | |
| - uses: actions/upload-artifact@v4 | |
| with: | |
| path: dist/*.whl | |
| name: artifact-mac-${{ matrix.arch }}-py${{ matrix.python-version }} | |
| if-no-files-found: error |