From 6d5ee1c6f26a4e0ca7c7e75e1335555e4cc7e7aa Mon Sep 17 00:00:00 2001 From: Lasse Blaauwbroek Date: Tue, 4 Apr 2023 03:55:17 +0200 Subject: [PATCH] Use cibuildwheel in ci (#309) * Use cibuildwheel in ci `cibuildwheel` is a system that automatically compiles and repairs wheels for many python versions and architectures at once. This has some advantages vs the old situation: - Macosx wheels had inconsistent minimum versions ranging between 10.9 and 11.0. I'm not sure why this happens, but for some users this means they have to build from source on macosx. With cibuildwheel, the build is consistent, with 10.9 as the minimum for x86 and 11.0 for arm64. - Consolidation between the packaging tests and manylinux tests. - Addition of musllinux targets and additional cross-compilation to ppc64le and s390x. - With cibuildwheel, new python versions should be automatically picked up. - Separation of the sdist build and lint checks. There is not reason to run that many times. All possible build targets succeed, except for ARM64 on Windows. The upstream capnp build fails. I've disabled it. The cross-compilation builds on linux are pretty slow. This could potentially be sped up by separating the builds of manylinux and musllinux, but I'm not sure if it's worth the extra complexity. (One can also contemplate disabling these targets.) Tests for macosx arm64 cannot be run (but also couldn't be run in the previous system. This should be remedied once apple silicon becomes available on the CI. I've also added some commented-out code that can automatically take care of uploading a build to PyPi when a release is created. One might contemplate using this. * Set CMAKE_OSX_ARCHITECTURES for arm64 and disable universal2 --- .github/workflows/manylinux2014.yml | 41 ---------- .github/workflows/packagingtest.yml | 82 -------------------- .github/workflows/wheels.yml | 116 ++++++++++++++++++++++++++++ setup.py | 1 + 4 files changed, 117 insertions(+), 123 deletions(-) delete mode 100644 .github/workflows/manylinux2014.yml delete mode 100644 .github/workflows/packagingtest.yml create mode 100644 .github/workflows/wheels.yml diff --git a/.github/workflows/manylinux2014.yml b/.github/workflows/manylinux2014.yml deleted file mode 100644 index 4fec5a8..0000000 --- a/.github/workflows/manylinux2014.yml +++ /dev/null @@ -1,41 +0,0 @@ -name: manylinux2014 - -on: [push, pull_request] - -jobs: - manylinux2014: - name: "manylinux2014( ${{ matrix.python-version }}, ${{ matrix.container-image }})" - runs-on: ubuntu-latest - strategy: - max-parallel: 99 - matrix: - python-version: ['cp37-cp37m', 'cp38-cp38', 'cp39-cp39', 'cp310-cp310', 'cp311-cp311'] - container-image: ['quay.io/pypa/manylinux2014_aarch64', 'quay.io/pypa/manylinux2014_x86_64', 'quay.io/pypa/manylinux2014_i686'] - fail-fast: false - env: - py: /opt/python/${{ matrix.python-version }}/bin/python - steps: - - uses: actions/checkout@v3 - - name: Set up QEMU - id: qemu - uses: docker/setup-qemu-action@v2 - - name: Building pycapnp and packaging - run: | - docker run --rm -v ${{ github.workspace }}:/ws:rw --workdir=/ws \ - ${{ matrix.container-image }} \ - bash -exc '${{ env.py }} -m venv .env && \ - source .env/bin/activate && \ - echo "Install Dependencies" && \ - python -m pip install -r requirements.txt && \ - python -m pip install auditwheel && \ - echo "Build pycapnp and install" && \ - python setup.py build && \ - echo "Packaging" && \ - python setup.py bdist_wheel && \ - python setup.py sdist && \ - auditwheel repair dist/*linux_*.whl && \ - deactivate' - - uses: actions/upload-artifact@v1.0.0 - with: - name: manylinux2014_dist - path: wheelhouse diff --git a/.github/workflows/packagingtest.yml b/.github/workflows/packagingtest.yml deleted file mode 100644 index b61d3c9..0000000 --- a/.github/workflows/packagingtest.yml +++ /dev/null @@ -1,82 +0,0 @@ -name: Packaging Test - -on: [push, pull_request] - -jobs: - build: - - runs-on: ${{ matrix.os }} - strategy: - max-parallel: 99 - fail-fast: false - matrix: - # Some asyncio commands require 3.7+ - # It may be possible to use 3.6 and maybe 3.5; however, this will take some patching to get examples to work - python-version: [3.7, 3.8, 3.9, "3.10", "3.11"] - os: [ubuntu-latest, macOS-latest, windows-2019] - arch: ["x86_64"] - include: - - python-version: "3.7" - os: macOS-latest - arch: arm64 - - python-version: "3.8" - os: macOS-latest - arch: arm64 - - python-version: "3.9" - os: macOS-latest - arch: arm64 - - python-version: "3.10" - os: macOS-latest - arch: arm64 - - python-version: "3.11" - os: macOS-latest - arch: arm64 - - steps: - - uses: actions/checkout@v3 - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v4 - with: - python-version: ${{ matrix.python-version }} - - name: Install dependencies - run: | - python -m pip install --upgrade pip - pip install -r requirements.txt - - name: Build pycapnp and install - if: matrix.arch == 'x86_64' - run: | - python setup.py build - pip install . - - name: Lint with flake8 and check black - run: | - pip install black flake8 - flake8 . --filename '*.py,*.pyx,*.pxd' --count --show-source --statistics --exclude benchmark,build,capnp/templates/module.pyx - flake8 . --count --show-source --statistics --exclude benchmark,build - black . --check --diff --color - - name: Packaging x86_64 - if: matrix.arch == 'x86_64' - run: | - python setup.py bdist_wheel - python setup.py sdist - - name: macOS - Packaging arm64 - if: matrix.os == 'macOS-latest' && matrix.arch == 'arm64' - env: - CMAKE_OSX_ARCHITECTURES: arm64 # capnp cmake - MACOSX_DEPLOYMENT_TARGET: "11.0" # python wheel - ARCHFLAGS: "-arch arm64" # python wheel - _PYTHON_HOST_PLATFORM: "macosx-11.0-arm64" # python wheel - run: | - env - rm -rf dist build build64 bundled - python setup.py bdist_wheel --force-bundled-libcapnp - ls -lh dist - python setup.py sdist - - uses: actions/upload-artifact@v1.0.0 - with: - name: package_dist - path: dist - - name: Test with pytest - if: matrix.arch == 'x86_64' - run: | - pip install pytest - pytest diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml new file mode 100644 index 0000000..76d1fd4 --- /dev/null +++ b/.github/workflows/wheels.yml @@ -0,0 +1,116 @@ +name: Build + +on: [push, pull_request] + +jobs: + build_wheels: + name: Build wheels on ${{ matrix.os }} ${{ matrix.arch }} + runs-on: ${{ matrix.os }} + strategy: + max-parallel: 99 + fail-fast: false + matrix: + include: + - os: ubuntu-latest + arch: x86_64 + - os: ubuntu-latest + arch: i686 + - os: ubuntu-latest + arch: aarch64 + - os: ubuntu-latest + arch: ppc64le + - os: ubuntu-latest + arch: s390x + + - os: macOS-latest + arch: x86_64 + - os: macOS-latest + arch: arm64 + # Disabled until someone figures out how to build capnproto for arm64 and x86_64 simultaneously + # - os: macOS-latest + # arch: universal2 + + - os: windows-2019 + arch: AMD64 + - os: windows-2019 + arch: x86 + # Does not build currently + # - os: windows-2019 + # arch: ARM64 + + steps: + - uses: actions/checkout@v3 + + - name: Set up QEMU + if: runner.os == 'Linux' + uses: docker/setup-qemu-action@v2 + + # Used to host cibuildwheel + - uses: actions/setup-python@v4 + with: + python-version: '3.11' + + - name: Install cibuildwheel + run: python -m pip install cibuildwheel==2.12.0 + + - name: Build wheels + run: python -m cibuildwheel --output-dir wheelhouse + env: + CIBW_ARCHS: ${{ matrix.arch }} + # TODO: Disable building PyPy wheels. If the build system gets modernized, this should be + # auto-detected based on the Cython dependency. + CIBW_SKIP: pp* + CIBW_TEST_REQUIRES: pytest + CIBW_TEST_COMMAND: pytest {project} + # Only needed to make the macosx arm64 build work + CMAKE_OSX_ARCHITECTURES: "${{ matrix.arch == 'arm64' && 'arm64' || '' }}" + + - uses: actions/upload-artifact@v3 + with: + path: ./wheelhouse/*.whl + + build_sdist: + name: Build source distribution + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + + - name: Build sdist + run: pipx run build --sdist + + - uses: actions/upload-artifact@v3 + with: + path: dist/*.tar.gz + + lint: + name: Lint with flake8 and check black + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + + - name: Lint with flake8 and check black + run: | + pip install black flake8 + flake8 . --filename '*.py,*.pyx,*.pxd' --count --show-source --statistics --exclude benchmark,build,capnp/templates/module.pyx + flake8 . --count --show-source --statistics --exclude benchmark,build + black . --check --diff --color + + # upload_pypi: + # needs: [build_wheels, build_sdist] + # runs-on: ubuntu-latest + # if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v') + # steps: + # - uses: actions/download-artifact@v3 + # with: + # # unpacks default artifact into dist/ + # # if `name: artifact` is omitted, the action will create extra parent dir + # name: artifact + # path: dist + + # - uses: pypa/gh-action-pypi-publish@v1.5.0 + # with: + # user: __token__ + # password: ${{ secrets.PYPI_PASSWORD_RELEASE }} + + # # password: ${{ secrets.PYPI_PASSWORD }} + # # repository_url: https://test.pypi.org/legacy/ diff --git a/setup.py b/setup.py index baa158b..c79a51c 100644 --- a/setup.py +++ b/setup.py @@ -208,6 +208,7 @@ extensions = [ ] setup( + python_requires=">=3.7", name="pycapnp", packages=["capnp"], version=VERSION,