Use cibuildwheel in ci (#309)

* Use cibuildwheel in ci

`cibuildwheel` is a system that automatically compiles and repairs wheels for
many python versions and architectures at once. This has some advantages vs the
old situation:
- Macosx wheels had inconsistent minimum versions ranging between 10.9 and
  11.0. I'm not sure why this happens, but for some users this means they have
  to build from source on macosx. With cibuildwheel, the build is
  consistent, with 10.9 as the minimum for x86 and 11.0 for arm64.
- Consolidation between the packaging tests and manylinux tests.
- Addition of musllinux targets and additional cross-compilation to ppc64le and
s390x.
- With cibuildwheel, new python versions should be automatically picked up.
- Separation of the sdist build and lint checks. There is not reason to run that
  many times.

All possible build targets succeed, except for ARM64 on Windows. The upstream
capnp build fails. I've disabled it.

The cross-compilation builds on linux are pretty slow. This could potentially be
sped up by separating the builds of manylinux and musllinux, but I'm not sure if
it's worth the extra complexity. (One can also contemplate disabling these
targets.)

Tests for macosx arm64 cannot be run (but also couldn't be run in the previous
system. This should be remedied once apple silicon becomes available on the CI.

I've also added some commented-out code that can automatically take care of
uploading a build to PyPi when a release is created. One might contemplate using this.

* Set CMAKE_OSX_ARCHITECTURES for arm64 and disable universal2

.github/workflows/manylinux2014.yml

@@ -1,41 +0,0 @@
-name: manylinux2014
-
-on: [push, pull_request]
-
-jobs:
-  manylinux2014:
-      name: "manylinux2014( ${{ matrix.python-version }}, ${{ matrix.container-image }})"
-      runs-on: ubuntu-latest
-      strategy:
-        max-parallel: 99
-        matrix:
-          python-version: ['cp37-cp37m', 'cp38-cp38', 'cp39-cp39', 'cp310-cp310', 'cp311-cp311']
-          container-image: ['quay.io/pypa/manylinux2014_aarch64', 'quay.io/pypa/manylinux2014_x86_64', 'quay.io/pypa/manylinux2014_i686']
-        fail-fast: false
-      env:
-        py: /opt/python/${{ matrix.python-version }}/bin/python
-      steps:
-      - uses: actions/checkout@v3
-      - name: Set up QEMU
-        id: qemu
-        uses: docker/setup-qemu-action@v2
-      - name: Building pycapnp and packaging
-        run: |
-          docker run --rm -v ${{ github.workspace }}:/ws:rw --workdir=/ws \
-          ${{ matrix.container-image }} \
-          bash -exc '${{ env.py }} -m venv .env && \
-          source .env/bin/activate && \
-          echo "Install Dependencies" && \
-          python -m pip install -r requirements.txt && \
-          python -m pip install auditwheel && \
-          echo "Build pycapnp and install" && \
-          python setup.py build && \
-          echo "Packaging" && \
-          python setup.py bdist_wheel && \
-          python setup.py sdist && \
-          auditwheel repair dist/*linux_*.whl && \
-          deactivate'
-      - uses: actions/upload-artifact@v1.0.0
-        with:
-          name: manylinux2014_dist
-          path: wheelhouse

.github/workflows/packagingtest.yml

@@ -1,82 +0,0 @@
-name: Packaging Test
-
-on: [push, pull_request]
-
-jobs:
-  build:
-
-    runs-on: ${{ matrix.os }}
-    strategy:
-      max-parallel: 99
-      fail-fast: false
-      matrix:
-        # Some asyncio commands require 3.7+
-        # It may be possible to use 3.6 and maybe 3.5; however, this will take some patching to get examples to work
-        python-version: [3.7, 3.8, 3.9, "3.10", "3.11"]
-        os: [ubuntu-latest, macOS-latest, windows-2019]
-        arch: ["x86_64"]
-        include:
-          - python-version: "3.7"
-            os: macOS-latest
-            arch: arm64
-          - python-version: "3.8"
-            os: macOS-latest
-            arch: arm64
-          - python-version: "3.9"
-            os: macOS-latest
-            arch: arm64
-          - python-version: "3.10"
-            os: macOS-latest
-            arch: arm64
-          - python-version: "3.11"
-            os: macOS-latest
-            arch: arm64
-
-    steps:
-    - uses: actions/checkout@v3
-    - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v4
-      with:
-        python-version: ${{ matrix.python-version }}
-    - name: Install dependencies
-      run: |
-        python -m pip install --upgrade pip
-        pip install -r requirements.txt
-    - name: Build pycapnp and install
-      if: matrix.arch == 'x86_64'
-      run: |
-        python setup.py build
-        pip install .
-    - name: Lint with flake8 and check black
-      run: |
-        pip install black flake8
-        flake8 . --filename '*.py,*.pyx,*.pxd' --count --show-source --statistics --exclude benchmark,build,capnp/templates/module.pyx
-        flake8 . --count --show-source --statistics --exclude benchmark,build
-        black . --check --diff --color
-    - name: Packaging x86_64
-      if: matrix.arch == 'x86_64'
-      run: |
-        python setup.py bdist_wheel
-        python setup.py sdist
-    - name: macOS - Packaging arm64
-      if: matrix.os == 'macOS-latest' && matrix.arch == 'arm64'
-      env:
-        CMAKE_OSX_ARCHITECTURES: arm64 # capnp cmake
-        MACOSX_DEPLOYMENT_TARGET: "11.0" # python wheel
-        ARCHFLAGS: "-arch arm64" # python wheel
-        _PYTHON_HOST_PLATFORM: "macosx-11.0-arm64" # python wheel
-      run: |
-        env
-        rm -rf dist build build64 bundled
-        python setup.py bdist_wheel --force-bundled-libcapnp
-        ls -lh dist
-        python setup.py sdist
-    - uses: actions/upload-artifact@v1.0.0
-      with:
-        name: package_dist
-        path: dist
-    - name: Test with pytest
-      if: matrix.arch == 'x86_64'
-      run: |
-        pip install pytest
-        pytest

.github/workflows/wheels.yml

@@ -0,0 +1,116 @@
+name: Build
+
+on: [push, pull_request]
+
+jobs:
+  build_wheels:
+    name: Build wheels on ${{ matrix.os }} ${{ matrix.arch }}
+    runs-on: ${{ matrix.os }}
+    strategy:
+      max-parallel: 99
+      fail-fast: false
+      matrix:
+        include:
+          - os: ubuntu-latest
+            arch: x86_64
+          - os: ubuntu-latest
+            arch: i686
+          - os: ubuntu-latest
+            arch: aarch64
+          - os: ubuntu-latest
+            arch: ppc64le
+          - os: ubuntu-latest
+            arch: s390x
+
+          - os: macOS-latest
+            arch: x86_64
+          - os: macOS-latest
+            arch: arm64
+          # Disabled until someone figures out how to build capnproto for arm64 and x86_64 simultaneously
+          # - os: macOS-latest
+          #   arch: universal2
+
+          - os: windows-2019
+            arch: AMD64
+          - os: windows-2019
+            arch: x86
+          # Does not build currently
+          # - os: windows-2019
+          #   arch: ARM64
+
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Set up QEMU
+        if: runner.os == 'Linux'
+        uses: docker/setup-qemu-action@v2
+
+      # Used to host cibuildwheel
+      - uses: actions/setup-python@v4
+        with:
+          python-version: '3.11'
+
+      - name: Install cibuildwheel
+        run: python -m pip install cibuildwheel==2.12.0
+
+      - name: Build wheels
+        run: python -m cibuildwheel --output-dir wheelhouse
+        env:
+          CIBW_ARCHS: ${{ matrix.arch }}
+          # TODO: Disable building PyPy wheels. If the build system gets modernized, this should be
+          # auto-detected based on the Cython dependency.
+          CIBW_SKIP: pp*
+          CIBW_TEST_REQUIRES: pytest
+          CIBW_TEST_COMMAND: pytest {project}
+          # Only needed to make the macosx arm64 build work
+          CMAKE_OSX_ARCHITECTURES: "${{ matrix.arch == 'arm64' && 'arm64' || '' }}"
+
+      - uses: actions/upload-artifact@v3
+        with:
+          path: ./wheelhouse/*.whl
+
+  build_sdist:
+    name: Build source distribution
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Build sdist
+        run: pipx run build --sdist
+
+      - uses: actions/upload-artifact@v3
+        with:
+          path: dist/*.tar.gz
+
+  lint:
+    name: Lint with flake8 and check black
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Lint with flake8 and check black
+        run: |
+          pip install black flake8
+          flake8 . --filename '*.py,*.pyx,*.pxd' --count --show-source --statistics --exclude benchmark,build,capnp/templates/module.pyx
+          flake8 . --count --show-source --statistics --exclude benchmark,build
+          black . --check --diff --color
+
+  # upload_pypi:
+  #   needs: [build_wheels, build_sdist]
+  #   runs-on: ubuntu-latest
+  #   if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v')
+  #   steps:
+  #     - uses: actions/download-artifact@v3
+  #       with:
+  #         # unpacks default artifact into dist/
+  #         # if `name: artifact` is omitted, the action will create extra parent dir
+  #         name: artifact
+  #         path: dist
+
+  #     - uses: pypa/gh-action-pypi-publish@v1.5.0
+  #       with:
+  #         user: __token__
+  #         password: ${{ secrets.PYPI_PASSWORD_RELEASE }}
+
+  #         # password: ${{ secrets.PYPI_PASSWORD }}
+  #         # repository_url: https://test.pypi.org/legacy/

setup.py

@@ -208,6 +208,7 @@ extensions = [
 ]
 
 setup(
+    python_requires=">=3.7",
     name="pycapnp",
     packages=["capnp"],
     version=VERSION,