mypyc build improvements (#3881)

Build in separate jobs. This makes it clearer if e.g. a single Python
version is failing. It also potentially gets you more parallelism.

Build everything on push to master.

Only build Linux 3.8 and 3.11 wheels on PRs.

.github/workflows/pypi_upload.yml

@@ -1,9 +1,12 @@
-name: Build wheels and publish to PyPI
+name: Build and publish
 
 on:
   release:
     types: [published]
   pull_request:
+  push:
+    branches:
+      - main
 
 permissions:
   contents: read
@@ -12,6 +15,7 @@ jobs:
   main:
     name: sdist + pure wheel
     runs-on: ubuntu-latest
+    if: github.event_name == 'release'
 
     steps:
       - uses: actions/checkout@v4
@@ -35,34 +39,58 @@ jobs:
           TWINE_PASSWORD: ${{ secrets.PYPI_TOKEN }}
         run: twine upload --verbose -u '__token__' dist/*
 
+  generate_wheels_matrix:
+    name: generate wheels matrix
+    runs-on: ubuntu-latest
+    outputs:
+      include: ${{ steps.set-matrix.outputs.include }}
+    steps:
+      - uses: actions/checkout@v3
+      - name: Install cibuildwheel and pypyp
+        run: |
+          pipx install cibuildwheel==2.15.0
+          pipx install pypyp==1
+      - name: generate matrix
+        if: github.event_name != 'pull_request'
+        run: |
+          {
+            cibuildwheel --print-build-identifiers --platform linux \
+            | pyp 'json.dumps({"only": x, "os": "ubuntu-latest"})' \
+            && cibuildwheel --print-build-identifiers --platform macos \
+            | pyp 'json.dumps({"only": x, "os": "macos-latest"})' \
+            && cibuildwheel --print-build-identifiers --platform windows \
+            | pyp 'json.dumps({"only": x, "os": "windows-latest"})'
+          } | pyp 'json.dumps(list(map(json.loads, lines)))' > /tmp/matrix
+        env:
+          CIBW_ARCHS_LINUX: x86_64
+          CIBW_ARCHS_MACOS: x86_64 arm64
+          CIBW_ARCHS_WINDOWS: AMD64
+      - name: generate matrix (PR)
+        if: github.event_name == 'pull_request'
+        run: |
+          cibuildwheel --print-build-identifiers --platform linux \
+          | pyp 'json.dumps({"only": x, "os": "ubuntu-latest"})' \
+          | pyp 'json.dumps(list(map(json.loads, lines)))' > /tmp/matrix
+        env:
+          CIBW_BUILD: "cp38-* cp311-*"
+          CIBW_ARCHS_LINUX: x86_64
+      - id: set-matrix
+        run: echo "include=$(cat /tmp/matrix)" | tee -a $GITHUB_OUTPUT
+
   mypyc:
-    name: mypyc wheels (${{ matrix.name }})
+    name: mypyc wheels ${{ matrix.only }}
+    needs: generate_wheels_matrix
     runs-on: ${{ matrix.os }}
     strategy:
       fail-fast: false
       matrix:
-        include:
-          - os: ubuntu-latest
-            name: linux-x86_64
-          - os: windows-2019
-            name: windows-amd64
-          - os: macos-11
-            name: macos-x86_64
-            macos_arch: "x86_64"
-          - os: macos-11
-            name: macos-arm64
-            macos_arch: "arm64"
-          - os: macos-11
-            name: macos-universal2
-            macos_arch: "universal2"
+        include: ${{ fromJson(needs.generate_wheels_matrix.outputs.include) }}
 
     steps:
       - uses: actions/checkout@v4
-
-      - name: Build wheels via cibuildwheel
-        uses: pypa/cibuildwheel@v2.15.0
-        env:
-          CIBW_ARCHS_MACOS: "${{ matrix.macos_arch }}"
+      - uses: pypa/cibuildwheel@v2.15.0
+        with:
+          only: ${{ matrix.only }}
 
       - name: Upload wheels as workflow artifacts
         uses: actions/upload-artifact@v3
@@ -80,6 +108,7 @@ jobs:
     name: Update stable branch
     needs: [main, mypyc]
     runs-on: ubuntu-latest
+    if: github.event_name == 'release'
     permissions:
       contents: write
 

pyproject.toml

@@ -145,8 +145,8 @@ build-verbosity = 1
 # - Python: CPython 3.8+ only
 # - Architecture (64-bit only): amd64 / x86_64, universal2, and arm64
 # - OS: Linux (no musl), Windows, and macOS
-build = "cp3*-*"
-skip = ["*-manylinux_i686", "*-musllinux_*", "*-win32", "pp-*", "cp312-*"]
+build = "cp3*"
+skip = ["*-manylinux_i686", "*-musllinux_*", "*-win32", "pp*", "cp312-*"]
 # This is the bare minimum needed to run the test suite. Pulling in the full
 # test_requirements.txt would download a bunch of other packages not necessary
 # here and would slow down the testing step a fair bit.