From 6c350d88fe165d3a7ceea5d004c4d6f8e09ccad6 Mon Sep 17 00:00:00 2001 From: Nicolas Patry Date: Wed, 23 Aug 2023 15:37:38 +0200 Subject: [PATCH] Re-using scritpts from safetensors. (#1328) --- .github/workflows/python-release.yml | 166 ++++++++++++++++----------- .github/workflows/rust-release.yml | 1 + 2 files changed, 100 insertions(+), 67 deletions(-) diff --git a/.github/workflows/python-release.yml b/.github/workflows/python-release.yml index 4252573d..39440f04 100644 --- a/.github/workflows/python-release.yml +++ b/.github/workflows/python-release.yml @@ -1,5 +1,4 @@ name: Python Release - on: push: tags: @@ -13,13 +12,21 @@ env: DIST_DIR: ${{ github.sha }} jobs: + lock_exists: + runs-on: ubuntu-latest + name: Cargo.lock + steps: + - uses: actions/checkout@v3 + - name: Cargo.lock lock exists + run: cat Cargo.lock + working-directory: ./bindings/python create_wheels_manylinux: runs-on: ubuntu-latest + needs: [lock_exists] name: Create wheels for manylinux2014 container: quay.io/pypa/manylinux2014_x86_64 steps: - # v1 is required when using manylinux2010 - uses: actions/checkout@v3 - name: Install dependencies @@ -29,29 +36,40 @@ jobs: working-directory: ./bindings/python run: sh build-wheels.sh - create_wheels_windows_32bit: - name: Create wheels for windows 32-bit + create_wheels_windows: + name: Windows runs-on: windows-latest + needs: [lock_exists] strategy: matrix: python: ["3.7", "3.8", "3.9", "3.10", "3.11"] + bits: ["32", "64"] steps: - name: Checkout repository uses: actions/checkout@v2 - - name: Install Rust + - name: Install Rust 32bits + if: ${{ matrix.os == '32' }} + uses: actions-rs/toolchain@v1 + with: + toolchain: stable-i686-pc-windows-msvc + override: true + + - name: Install Rust 64bits + if: ${{ matrix.os == '32' }} uses: actions-rs/toolchain@v1 with: toolchain: stable-i686-pc-windows-msvc override: true - name: Override toolchain + if: ${{ matrix.os == '32' }} shell: bash working-directory: ./bindings/python run: echo "stable-i686-pc-windows-msvc" > rust-toolchain - name: Install Python - uses: actions/setup-python@v1 + uses: actions/setup-python@v4 with: python-version: ${{ matrix.python }} architecture: x86 @@ -59,7 +77,7 @@ jobs: - name: Install dependencies run: | # On old versions of python there is an old version of setuptools already installed - pip install setuptools wheel setuptools-rust==0.11.3 --ignore-installed --force-reinstall + pip install setuptools wheel setuptools-rust --ignore-installed --force-reinstall - name: Build wheel working-directory: ./bindings/python @@ -71,16 +89,67 @@ jobs: pip install awscli aws s3 sync --exact-timestamps ./bindings/python/dist "s3://tokenizers-releases/python/$DIST_DIR" - - create_wheels_others_64bit: - name: Other OSes - env: - MACOSX_DEPLOYMENT_TARGET: 10.11 + create_wheels_macos_conda: + name: MacOS - Conda runs-on: ${{ matrix.os }} + needs: [lock_exists] + strategy: + matrix: + os: [macos-latest] + # 3.11 not available on Conda yet. + python: ["3.7", "3.8", "3.9", "3.10"] + + steps: + - name: Checkout repository + uses: actions/checkout@v3 + + - name: Install miniconda + uses: conda-incubator/setup-miniconda@v2 + with: + auto-update-conda: true + python-version: ${{ matrix.python }} + + - name: Conda info + shell: bash -l {0} + run: conda info + + - name: Install Rust + uses: actions-rs/toolchain@v1 + with: + toolchain: stable + + - name: Setup conda env + shell: bash -l {0} + run: | + conda install setuptools-rust + conda install -c defaults anaconda-client conda-build + + - name: Extract version + shell: bash -l {0} + working-directory: ./bindings/python + run: echo "TOKENIZERS_VERSION=`python setup.py --version`" >> $GITHUB_ENV + + - name: Build conda packages + shell: bash -l {0} + working-directory: ./bindings/python + run: | + MACOSX_DEPLOYMENT_TARGET=10.11 python setup.py bdist_wheel + + - name: Upload wheels + shell: bash + run: | + pip install awscli + aws s3 sync --exact-timestamps ./bindings/python/dist "s3://tokenizers-releases/python/$DIST_DIR" + + create_wheels_macos: + name: MacOS + runs-on: ${{ matrix.os.os }} + needs: [lock_exists] strategy: matrix: - os: [windows-latest, macos-latest] python: ["3.7", "3.8", "3.9", "3.10", "3.11"] + # os: [{os: "macos-11", target: "11.0"}, {os: "macos-12"}, {os: "macos-13"}, {os: "macos-13", target: "14.0"}] + os: [{os: "macos-11", target: "11.0"}, {os: "macos-12"}, {os: "macos-13"}] steps: - name: Checkout repository uses: actions/checkout@v2 @@ -95,21 +164,20 @@ jobs: uses: actions/setup-python@v4 with: python-version: ${{ matrix.python }} - architecture: x64 - name: Install dependencies run: | # On old versions of python there is an old version of setuptools already installed - pip install setuptools wheel setuptools-rust==0.11.3 --ignore-installed --force-reinstall + pip install setuptools wheel setuptools-rust --ignore-installed --force-reinstall + + - name: Override target + if: ${{ matrix.os.target }} + working-directory: ./bindings/python + run: echo "MACOSX_DEPLOYMENT_TARGET=${{ matrix.os.target }}" >> $GITHUB_ENV - name: Build wheel working-directory: ./bindings/python - run: python setup.py bdist_wheel - - - name: Rename wheels - shell: bash - working-directory: ./bindings/python/dist - run: for file in *.whl ; do mv $file ${file//macosx_10_1[0-9]/macosx_10_11} || true; done + run: echo $MACOX_DEPLOYMENT_TARGET && python setup.py bdist_wheel - name: Upload wheels shell: bash @@ -117,54 +185,15 @@ jobs: pip install awscli aws s3 sync --exact-timestamps ./bindings/python/dist "s3://tokenizers-releases/python/$DIST_DIR" - create_wheels_macos_13: - name: MacOS 13 - runs-on: ${{ matrix.os }} - strategy: - matrix: - os: [macos-13] - python: ["3.7", "3.8", "3.9", "3.10", "3.11"] - steps: - - name: Checkout repository - uses: actions/checkout@v2 - - - name: Install Rust - uses: actions-rs/toolchain@v1 - with: - toolchain: stable - override: true - - - name: Install Python - uses: actions/setup-python@v4 - with: - python-version: ${{ matrix.python }} - architecture: x64 - - - name: Install dependencies - run: | - # On old versions of python there is an old version of setuptools already installed - pip install setuptools wheel setuptools-rust==0.11.3 --ignore-installed --force-reinstall - - name: Build wheel - working-directory: ./bindings/python - run: python setup.py bdist_wheel - - - name: Rename wheels - shell: bash - working-directory: ./bindings/python/dist - run: for file in *.whl ; do mv $file ${file//macosx_10_1[0-9]/macosx_10_11} || true; done - - - name: Upload wheels - shell: bash - run: | - pip install awscli - aws s3 sync --exact-timestamps ./bindings/python/dist "s3://tokenizers-releases/python/$DIST_DIR" create_wheels_macos_arm64: - name: Create wheels for MacOS M1 + name: MacOS M1 runs-on: macos-arm64 + needs: [lock_exists] strategy: matrix: - python: [ "3.9.13", "3.10.6", "3.11.1"] + python: ["3.9.13", "3.10.6", "3.11.0"] + # target: ["12.0", "13.0", "14.0"] target: ["12.0", "13.0"] steps: - name: Checkout repository @@ -192,16 +221,19 @@ jobs: cd ../../ aws s3 sync --exact-timestamps ./bindings/python/dist "s3://tokenizers-releases/python/$DIST_DIR" - upload_package: + Upload_package: name: Upload package to PyPi runs-on: ubuntu-latest - needs: [create_wheels_manylinux, create_wheels_windows_32bit, create_wheels_others_64bit, create_wheels_macos_arm64, create_wheels_macos_13] + needs: [create_wheels_manylinux, create_wheels_windows, create_wheels_macos, create_wheels_macos_arm64, create_wheels_macos_conda] steps: - uses: actions/checkout@v3 - name: Install Python - uses: actions/setup-python@v1 + uses: actions/setup-python@v4 + with: + python-version: "3.10" + architecture: x64 - name: Retrieve all wheels shell: bash diff --git a/.github/workflows/rust-release.yml b/.github/workflows/rust-release.yml index 6f52363d..2095a719 100644 --- a/.github/workflows/rust-release.yml +++ b/.github/workflows/rust-release.yml @@ -28,5 +28,6 @@ jobs: - name: Publish package rust working-directory: ./tokenizers + if: ${{ !contains(github.ref, 'rc') }} run: cargo publish --token ${CRATES_TOKEN}