From 2bea383f2cdbcf805262c2561282aaf58b53a263 Mon Sep 17 00:00:00 2001 From: Sviatoslav Sydorenko Date: Tue, 28 May 2024 21:23:54 +0200 Subject: [PATCH 1/3] =?UTF-8?q?=F0=9F=90=9B=F0=9F=A7=AA=20Fix=20referencin?= =?UTF-8?q?g=20GHA=20artifact=20@=20whl=20builders?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit It wasn't obvious right away, but referencing a job in the `${{ needs }}` context that is not being depended on, and does not even exist, silently returned an empty string for the whole expression passed to the `name:` input of the `download-artifact` action invocations. This subsequently triggered the behavior of said action to change to "download all the artifacts that exist in the workflow". This "download all" behavior has an additional quirk, though — it downloads each artifact in a subdirectory named after the artifact name. The behavior of the "download one specific artifact" mode, on the other hand, is different in that it does not create an extra level of nesting. This was the reason why it felt necessary to unpack two-levels deep sdist tarballs in the first place. With this patch, references to said sdists will change. --- .github/workflows/ci.yaml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 3ba89a8b..3b6417ac 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -29,6 +29,8 @@ env: jobs: python_sdist: runs-on: ubuntu-22.04 + outputs: + artifact_name: ${{ steps.build_sdist.outputs.artifact_name }} steps: - name: clone repo uses: actions/checkout@v4 @@ -258,7 +260,7 @@ jobs: id: fetch_sdist uses: actions/download-artifact@v4 with: - name: ${{ needs.build_sdist.outputs.artifact_name }} + name: ${{ needs.python_sdist.outputs.artifact_name }} - name: configure docker foreign arch support uses: docker/setup-qemu-action@v3 @@ -389,7 +391,7 @@ jobs: id: fetch_sdist uses: actions/download-artifact@v4 with: - name: ${{ needs.build_sdist.outputs.artifact_name }} + name: ${{ needs.python_sdist.outputs.artifact_name }} - name: install python uses: actions/setup-python@v5 @@ -492,7 +494,7 @@ jobs: id: fetch_sdist uses: actions/download-artifact@v4 with: - name: ${{ needs.build_sdist.outputs.artifact_name }} + name: ${{ needs.python_sdist.outputs.artifact_name }} - name: build/test wheels id: build From 4e60e2a6aa188f7b81787131a9daa90bc88c5910 Mon Sep 17 00:00:00 2001 From: Sviatoslav Sydorenko Date: Tue, 28 May 2024 21:02:02 +0200 Subject: [PATCH 2/3] =?UTF-8?q?=F0=9F=92=85=20Move=20whl=20name=20lookup?= =?UTF-8?q?=20to=20dedicated=20GHA=20job=20step?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/ci.yaml | 38 ++++++++++++++++++++++++-------------- 1 file changed, 24 insertions(+), 14 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 3b6417ac..50da7842 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -267,7 +267,6 @@ jobs: if: ${{ ! contains(matrix.spec, 'x86_64') }} - name: build/test wheels - id: build env: CFLAGS: -Dffi_call=cffistatic_ffi_call # override name for ffi_call to break hard if we linked against someone else's libffi CIBW_ARCHS_LINUX: all @@ -304,13 +303,18 @@ jobs: # actually build libffi + wheel (using env tweaks above) python -m cibuildwheel --output-dir dist ./cffi - echo "artifact_name=$(ls ./dist/)" >> "$GITHUB_OUTPUT" + shell: bash + + - name: determine built wheel filename + id: built-artifact-lookup + run: echo "artifact_name=$(ls ./dist/)" >> "${GITHUB_OUTPUT}" + shell: bash -eEuxo pipefail {0} - name: upload artifacts uses: actions/upload-artifact@v4 with: - name: ${{ steps.build.outputs.artifact_name }} - path: dist/*.whl + name: ${{ steps.built-artifact-lookup.outputs.artifact_name }} + path: dist/${{ steps.built-artifact-lookup.outputs.artifact_name }} if-no-files-found: error if: ${{ env.skip_artifact_upload != 'true' }} @@ -406,7 +410,6 @@ jobs: brew uninstall --ignore-dependencies libffi 2>&1 || true - name: build/test wheels - id: build env: CIBW_BUILD: ${{ matrix.spec }} CIBW_PRERELEASE_PYTHONS: 'True' @@ -422,14 +425,19 @@ jobs: tar zxf ${{ steps.fetch_sdist.outputs.download-path }}/cffi*.tar.gz/cffi*.tar.gz --strip-components=1 -C cffi python3 -m cibuildwheel --output-dir dist cffi - - echo "artifact_name=$(ls ./dist/)" >> "$GITHUB_OUTPUT" + + shell: bash + + - name: determine built wheel filename + id: built-artifact-lookup + run: echo "artifact_name=$(ls ./dist/)" >> "${GITHUB_OUTPUT}" + shell: bash -eEuxo pipefail {0} - name: upload artifacts uses: actions/upload-artifact@v4 with: - name: ${{ steps.build.outputs.artifact_name }} - path: dist/*.whl + name: ${{ steps.built-artifact-lookup.outputs.artifact_name }} + path: dist/${{ steps.built-artifact-lookup.outputs.artifact_name }} if-no-files-found: error if: ${{ env.skip_artifact_upload != 'true' }} @@ -497,7 +505,6 @@ jobs: name: ${{ needs.python_sdist.outputs.artifact_name }} - name: build/test wheels - id: build env: CIBW_BUILD: ${{ matrix.spec }} CIBW_PRERELEASE_PYTHONS: 'True' @@ -515,16 +522,19 @@ jobs: python -m pip install --upgrade pip pip install "${{ matrix.cibw_version || 'cibuildwheel'}}" python -m cibuildwheel --output-dir dist cffi - - echo "artifact_name=$(ls ./dist/)" >> "$GITHUB_OUTPUT" shell: bash + - name: determine built wheel filename + id: built-artifact-lookup + run: echo "artifact_name=$(ls ./dist/)" >> "${GITHUB_OUTPUT}" + shell: bash -eEuxo pipefail {0} + - name: upload artifacts uses: actions/upload-artifact@v4 with: - name: ${{ steps.build.outputs.artifact_name }} - path: dist/*.whl + name: ${{ steps.built-artifact-lookup.outputs.artifact_name }} + path: dist/${{ steps.built-artifact-lookup.outputs.artifact_name }} if-no-files-found: error if: ${{ env.skip_artifact_upload != 'true' }} From 2eda2d149385a6395fcd0e7f8a0e23a453eaa0c0 Mon Sep 17 00:00:00 2001 From: Sviatoslav Sydorenko Date: Tue, 28 May 2024 21:15:14 +0200 Subject: [PATCH 3/3] =?UTF-8?q?=F0=9F=A7=AA=20Pass=20sdist=20into=20`cibui?= =?UTF-8?q?ldwheel`=20directly?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This patch also brings a copy of `cibuildwheel` in through the GitHub Actions interface instead of PyPI. --- .github/workflows/ci.yaml | 67 ++++++++++++++------------------------- 1 file changed, 23 insertions(+), 44 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 50da7842..0b9882d8 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -267,6 +267,7 @@ jobs: if: ${{ ! contains(matrix.spec, 'x86_64') }} - name: build/test wheels + uses: pypa/cibuildwheel@v2.18.1 env: CFLAGS: -Dffi_call=cffistatic_ffi_call # override name for ffi_call to break hard if we linked against someone else's libffi CIBW_ARCHS_LINUX: all @@ -292,29 +293,21 @@ jobs: CIBW_PRERELEASE_PYTHONS: 'True' CIBW_TEST_REQUIRES: pytest setuptools # 3.12+ no longer includes distutils, just always ensure setuptools is present CIBW_TEST_COMMAND: PYTHONUNBUFFERED=1 python -m pytest ${{ matrix.test_args || '{project}' }} # default to test all - run: | - set -eux - - mkdir cffi - - tar zxf ${{ steps.fetch_sdist.outputs.download-path }}/cffi*.tar.gz/cffi*.tar.gz --strip-components=1 -C cffi - python -m pip install --upgrade "${{ matrix.cibw_version || 'cibuildwheel' }}" - - # actually build libffi + wheel (using env tweaks above) - python -m cibuildwheel --output-dir dist ./cffi - - shell: bash + with: + package-dir: >- + ${{ steps.fetch_sdist.outputs.download-path + }}/${{ needs.python_sdist.outputs.artifact_name }} - name: determine built wheel filename id: built-artifact-lookup - run: echo "artifact_name=$(ls ./dist/)" >> "${GITHUB_OUTPUT}" + run: echo "artifact_name=$(ls ./wheelhouse/)" >> "${GITHUB_OUTPUT}" shell: bash -eEuxo pipefail {0} - name: upload artifacts uses: actions/upload-artifact@v4 with: name: ${{ steps.built-artifact-lookup.outputs.artifact_name }} - path: dist/${{ steps.built-artifact-lookup.outputs.artifact_name }} + path: wheelhouse/${{ steps.built-artifact-lookup.outputs.artifact_name }} if-no-files-found: error if: ${{ env.skip_artifact_upload != 'true' }} @@ -404,12 +397,11 @@ jobs: # built-in virtualenv/pip are pinned to busted versions that fail on newer Pythons - name: build wheel prereqs - run: | - set -eux - python3 -m pip install --user --upgrade "${{ matrix.cibw_version || 'cibuildwheel' }}" - brew uninstall --ignore-dependencies libffi 2>&1 || true + run: brew uninstall --ignore-dependencies libffi 2>&1 || true + shell: bash -eux {0} - name: build/test wheels + uses: pypa/cibuildwheel@v2.18.1 env: CIBW_BUILD: ${{ matrix.spec }} CIBW_PRERELEASE_PYTHONS: 'True' @@ -417,27 +409,21 @@ jobs: CIBW_TEST_COMMAND: pip install pip --upgrade; cd {project}; PYTHONUNBUFFERED=1 pytest MACOSX_DEPLOYMENT_TARGET: ${{ matrix.deployment_target || '10.9' }} SDKROOT: ${{ matrix.sdkroot || 'macosx' }} - run: | - set -eux - - mkdir cffi - - tar zxf ${{ steps.fetch_sdist.outputs.download-path }}/cffi*.tar.gz/cffi*.tar.gz --strip-components=1 -C cffi - - python3 -m cibuildwheel --output-dir dist cffi - - shell: bash + with: + package-dir: >- + ${{ steps.fetch_sdist.outputs.download-path + }}/${{ needs.python_sdist.outputs.artifact_name }} - name: determine built wheel filename id: built-artifact-lookup - run: echo "artifact_name=$(ls ./dist/)" >> "${GITHUB_OUTPUT}" + run: echo "artifact_name=$(ls ./wheelhouse/)" >> "${GITHUB_OUTPUT}" shell: bash -eEuxo pipefail {0} - name: upload artifacts uses: actions/upload-artifact@v4 with: name: ${{ steps.built-artifact-lookup.outputs.artifact_name }} - path: dist/${{ steps.built-artifact-lookup.outputs.artifact_name }} + path: wheelhouse/${{ steps.built-artifact-lookup.outputs.artifact_name }} if-no-files-found: error if: ${{ env.skip_artifact_upload != 'true' }} @@ -505,6 +491,7 @@ jobs: name: ${{ needs.python_sdist.outputs.artifact_name }} - name: build/test wheels + uses: pypa/cibuildwheel@v2.18.1 env: CIBW_BUILD: ${{ matrix.spec }} CIBW_PRERELEASE_PYTHONS: 'True' @@ -512,29 +499,21 @@ jobs: CIBW_TEST_COMMAND: 'python -m pytest {package}/src/c' # FIXME: /testing takes ~45min on Windows and has some failures... # CIBW_TEST_COMMAND='python -m pytest {package}/src/c {project}/testing' - run: | - set -eux - - mkdir cffi - - tar zxf cffi*.tar.gz/cffi*.tar.gz --strip-components=1 -C cffi - - python -m pip install --upgrade pip - pip install "${{ matrix.cibw_version || 'cibuildwheel'}}" - python -m cibuildwheel --output-dir dist cffi - - shell: bash + with: + package-dir: >- + ${{ steps.fetch_sdist.outputs.download-path + }}/${{ needs.python_sdist.outputs.artifact_name }} - name: determine built wheel filename id: built-artifact-lookup - run: echo "artifact_name=$(ls ./dist/)" >> "${GITHUB_OUTPUT}" + run: echo "artifact_name=$(ls ./wheelhouse/)" >> "${GITHUB_OUTPUT}" shell: bash -eEuxo pipefail {0} - name: upload artifacts uses: actions/upload-artifact@v4 with: name: ${{ steps.built-artifact-lookup.outputs.artifact_name }} - path: dist/${{ steps.built-artifact-lookup.outputs.artifact_name }} + path: wheelhouse/${{ steps.built-artifact-lookup.outputs.artifact_name }} if-no-files-found: error if: ${{ env.skip_artifact_upload != 'true' }}