Moving to manylinux2010 and remove nightly on Windows. (#455)

* Moving to manylinux2010 and remove nightly on Windows. * Add build for manylinux2014 for powerpc and aarch64 + Python v3.9 * Also add support for IBM mainframe * try with env variables * Move extra builds to their own workflow Co-authored-by: Anthony MOI <m.anthony.moi@gmail.com>
2025-08-22 16:25:30 +00:00 · 2020-11-10 05:23:07 +01:00
parent b0d8108dcb
commit b122737ec6
3 changed files with 132 additions and 16 deletions
--- a/.github/workflows/python-release-extra.yml
+++ b/.github/workflows/python-release-extra.yml
@ -0,0 +1,115 @@
+name: Python Release extra
+
+on:
+  push:
+    tags:
+      - python-v*
+
+env:
+  AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+  AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+  AWS_DEFAULT_REGION: us-east-1
+  PYPI_TOKEN: ${{ secrets.PYPI_TOKEN_DIST }}
+  DIST_DIR: '${{ github.sha }}_extra'
+
+jobs:
+
+  create_wheels_manylinux_2014_ppc64le:
+    runs-on: ubuntu-latest
+    name: Create wheels for manylinux2014 - PowerPC
+    steps:
+      - uses: actions/checkout@v1
+
+      - name: Pull images
+        run: |
+          docker pull multiarch/qemu-user-static
+          docker pull quay.io/pypa/manylinux2014_ppc64le:latest
+
+      - name: Install QEMU
+        run: |
+          docker run --rm --privileged multiarch/qemu-user-static --reset -p yes
+
+      - name: Build and audit wheels
+        working-directory: ./bindings/python
+        run: |
+          docker run -e AWS_ACCESS_KEY_ID -e AWS_SECRET_ACCESS_KEY -e AWS_DEFAULT_REGION -e DIST_DIR \
+            --rm -v `pwd`/../..:/io quay.io/pypa/manylinux2014_ppc64le \
+            /bin/bash -c "cd /io/bindings/python; sh build-wheels.sh"
+
+  create_wheels_manylinux_2014_aarch64:
+    runs-on: ubuntu-latest
+    name: Create wheels for manylinux2014 - Aarch64
+    steps:
+      - uses: actions/checkout@v1
+
+      - name: Pull images
+        run: |
+          docker pull multiarch/qemu-user-static
+          docker pull quay.io/pypa/manylinux2014_aarch64:latest
+
+      - name: Install QEMU
+        run: |
+          docker run --rm --privileged multiarch/qemu-user-static --reset -p yes
+
+      - name: Build and audit wheels
+        working-directory: ./bindings/python
+        run: |
+          docker run -e AWS_ACCESS_KEY_ID -e AWS_SECRET_ACCESS_KEY -e AWS_DEFAULT_REGION -e DIST_DIR \
+            --rm -v `pwd`/../..:/io quay.io/pypa/manylinux2014_aarch64 \
+            /bin/bash -c "cd /io/bindings/python; sh build-wheels.sh"
+
+  create_wheels_manylinux_2014_x390x:
+    runs-on: ubuntu-latest
+    name: Create wheels for manylinux2014 - S390X
+    steps:
+      - uses: actions/checkout@v1
+
+      - name: Pull images
+        run: |
+          docker pull multiarch/qemu-user-static
+          docker pull quay.io/pypa/manylinux2014_s390x:latest
+
+      - name: Install QEMU
+        run: |
+          docker run --rm --privileged multiarch/qemu-user-static --reset -p yes
+
+      - name: Build and audit wheels
+        working-directory: ./bindings/python
+        run: |
+          docker run -e AWS_ACCESS_KEY_ID -e AWS_SECRET_ACCESS_KEY -e AWS_DEFAULT_REGION -e DIST_DIR \
+            --rm -v `pwd`/../..:/io quay.io/pypa/manylinux2014_s390x \
+            /bin/bash -c "cd /io/bindings/python; sh build-wheels.sh"
+
+  upload_package:
+    name: Upload package to PyPi
+    runs-on: ubuntu-latest
+    needs:
+      - create_wheels_manylinux_2014_ppc64le
+      - create_wheels_manylinux_2014_aarch64
+      - create_wheels_manylinux_2014_x390x
+
+    steps:
+      - uses: actions/checkout@v1
+
+      - name: Install Python
+        uses: actions/setup-python@v1
+
+      - name: Retrieve all wheels
+        shell: bash
+        run: |
+          pip install awscli
+          aws s3 sync "s3://tokenizers-releases/python/$DIST_DIR" ./bindings/python/dist
+
+      - name: Install dependencies
+        run: |
+          pip install setuptools wheel setuptools-rust
+
+      - name: Create source distribution
+        working-directory: ./bindings/python
+        run: sh build-sdist.sh
+
+      - name: Upload to PyPi
+        working-directory: ./bindings/python
+        run: |
+          pip install twine
+          twine upload dist/* -u __token__ -p "$PYPI_TOKEN"
--- a/.github/workflows/python-release.yml
+++ b/.github/workflows/python-release.yml
@ -10,13 +10,14 @@ env:
  AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
  AWS_DEFAULT_REGION: us-east-1
  PYPI_TOKEN: ${{ secrets.PYPI_TOKEN_DIST }}
+  DIST_DIR: ${{ github.sha }}

 jobs:

  create_wheels_manylinux:
    runs-on: ubuntu-latest
-    name: Create wheels for manylinux
-    container: quay.io/pypa/manylinux1_x86_64
+    name: Create wheels for manylinux2010
+    container: quay.io/pypa/manylinux2010_x86_64
    steps:
      - uses: actions/checkout@v1

@ -29,7 +30,7 @@ jobs:
    runs-on: windows-latest
    strategy:
      matrix:
-        python: [3.5, 3.6, 3.7, 3.8]
+        python: [3.5, 3.6, 3.7, 3.8, 3.9]
    steps:
      - name: Checkout repository
        uses: actions/checkout@v1
@ -37,13 +38,13 @@ jobs:
      - name: Install Rust
        uses: actions-rs/toolchain@v1
        with:
-          toolchain: nightly-2020-05-14-i686-pc-windows-msvc
+          toolchain: stable-i686-pc-windows-msvc
          override: true

      - name: Override toolchain
        shell: bash
        working-directory: ./bindings/python
-        run: echo "nightly-2020-05-14-i686-pc-windows-msvc" > rust-toolchain
+        run: echo "stable-i686-pc-windows-msvc" > rust-toolchain

      - name: Install Python
        uses: actions/setup-python@v1
@ -54,7 +55,7 @@ jobs:
      - name: Install dependencies
        run: |
          # On old versions of python there is an old version of setuptools already installed
-          pip install setuptools wheel setuptools-rust --ignore-installed --force-reinstall
+          pip install setuptools wheel setuptools-rust==0.11.3 --ignore-installed --force-reinstall

      - name: Build wheel
        working-directory: ./bindings/python
@ -64,7 +65,7 @@ jobs:
        shell: bash
        run: |
          pip install awscli
-          aws s3 sync --exact-timestamps ./bindings/python/dist "s3://tokenizers-releases/python/$GITHUB_SHA"
+          aws s3 sync --exact-timestamps ./bindings/python/dist "s3://tokenizers-releases/python/$DIST_DIR"


  create_wheels_others_64bit:
@ -75,7 +76,7 @@ jobs:
    strategy:
      matrix:
        os: [windows-latest, macos-10.15]
-        python: [3.5, 3.6, 3.7, 3.8]
+        python: [3.5, 3.6, 3.7, 3.8, 3.9]
    steps:
      - name: Checkout repository
        uses: actions/checkout@v1
@ -83,7 +84,7 @@ jobs:
      - name: Install Rust
        uses: actions-rs/toolchain@v1
        with:
-          toolchain: nightly-2020-05-18
+          toolchain: stable
          override: true

      - name: Install Python
@ -95,7 +96,7 @@ jobs:
      - name: Install dependencies
        run: |
          # On old versions of python there is an old version of setuptools already installed
-          pip install setuptools wheel setuptools-rust --ignore-installed --force-reinstall
+          pip install setuptools wheel setuptools-rust==0.11.3 --ignore-installed --force-reinstall

      - name: Build wheel
        working-directory: ./bindings/python
@ -110,7 +111,7 @@ jobs:
        shell: bash
        run: |
          pip install awscli
-          aws s3 sync --exact-timestamps ./bindings/python/dist "s3://tokenizers-releases/python/$GITHUB_SHA"
+          aws s3 sync --exact-timestamps ./bindings/python/dist "s3://tokenizers-releases/python/$DIST_DIR"

  upload_package:
    name: Upload package to PyPi
@ -127,7 +128,7 @@ jobs:
        shell: bash
        run: |
          pip install awscli
-          aws s3 sync "s3://tokenizers-releases/python/$GITHUB_SHA" ./bindings/python/dist
+          aws s3 sync "s3://tokenizers-releases/python/$DIST_DIR" ./bindings/python/dist

      - name: Install dependencies
        run: |
--- a/bindings/python/build-wheels.sh
+++ b/bindings/python/build-wheels.sh
@ -1,13 +1,13 @@
 #!/bin/bash
 set -ex

-curl https://sh.rustup.rs -sSf | sh -s -- --default-toolchain "1.46.0" -y
+curl https://sh.rustup.rs -sSf | sh -s -- -y
 export PATH="$HOME/.cargo/bin:$PATH"

-for PYBIN in /opt/python/{cp35-cp35m,cp36-cp36m,cp37-cp37m,cp38-cp38}/bin; do
+for PYBIN in /opt/python/{cp35-cp35m,cp36-cp36m,cp37-cp37m,cp38-cp38,cp39-cp39}/bin; do
    export PYTHON_SYS_EXECUTABLE="$PYBIN/python"

-    "${PYBIN}/pip" install -U setuptools-rust
+    "${PYBIN}/pip" install -U setuptools-rust==0.11.3
    "${PYBIN}/python" setup.py bdist_wheel
    rm -rf build/*
 done
@ -21,4 +21,4 @@ rm dist/*-linux_*

 # Upload wheels
 /opt/python/cp37-cp37m/bin/pip install -U awscli
-/opt/python/cp37-cp37m/bin/python -m awscli s3 sync --exact-timestamps ./dist "s3://tokenizers-releases/python/$GITHUB_SHA"
+/opt/python/cp37-cp37m/bin/python -m awscli s3 sync --exact-timestamps ./dist "s3://tokenizers-releases/python/$DIST_DIR"