tokenizers/.github/workflows/python-release.yml

name: Python Release

on:
  push:
    tags:
      - v*

env:
  AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
  AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
  AWS_DEFAULT_REGION: us-east-1
  PYPI_TOKEN: ${{ secrets.PYPI_TOKEN_DIST }}
  DIST_DIR: ${{ github.sha }}

jobs:

  create_wheels_manylinux:
    runs-on: ubuntu-latest
    name: Create wheels for manylinux2014
    container: quay.io/pypa/manylinux2014_x86_64
    steps:
      # v1 is required when using manylinux2010
      - uses: actions/checkout@v3

      - name: Install dependencies
        run: yum install -y openssl-devel

      - name: Build and audit wheels
        working-directory: ./bindings/python
        run: sh build-wheels.sh

  create_wheels_windows_32bit:
    name: Create wheels for windows 32-bit
    runs-on: windows-latest
    strategy:
      matrix:
        python: ["3.7", "3.8", "3.9", "3.10", "3.11"]
    steps:
      - name: Checkout repository
        uses: actions/checkout@v2

      - name: Install Rust
        uses: actions-rs/toolchain@v1
        with:
          toolchain: stable-i686-pc-windows-msvc
          override: true

      - name: Override toolchain
        shell: bash
        working-directory: ./bindings/python
        run: echo "stable-i686-pc-windows-msvc" > rust-toolchain

      - name: Install Python
        uses: actions/setup-python@v1
        with:
          python-version: ${{ matrix.python }}
          architecture: x86

      - name: Install dependencies
        run: |
          # On old versions of python there is an old version of setuptools already installed
          pip install setuptools wheel setuptools-rust==0.11.3 --ignore-installed --force-reinstall

      - name: Build wheel
        working-directory: ./bindings/python
        run: python setup.py bdist_wheel

      - name: Upload wheels
        shell: bash
        run: |
          pip install awscli
          aws s3 sync --exact-timestamps ./bindings/python/dist "s3://tokenizers-releases/python/$DIST_DIR"


  create_wheels_others_64bit:
    name: Other OSes
    env:
      MACOSX_DEPLOYMENT_TARGET: 10.11
    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        os: [windows-latest, macos-latest]
        python: ["3.7", "3.8", "3.9", "3.10", "3.11"]
    steps:
      - name: Checkout repository
        uses: actions/checkout@v2

      - name: Install Rust
        uses: actions-rs/toolchain@v1
        with:
          toolchain: stable
          override: true

      - name: Install Python
        uses: actions/setup-python@v4
        with:
          python-version: ${{ matrix.python }}
          architecture: x64

      - name: Install dependencies
        run: |
          # On old versions of python there is an old version of setuptools already installed
          pip install setuptools wheel setuptools-rust==0.11.3 --ignore-installed --force-reinstall

      - name: Build wheel
        working-directory: ./bindings/python
        run: python setup.py bdist_wheel

      - name: Rename wheels
        shell: bash
        working-directory: ./bindings/python/dist
        run: for file in *.whl ; do mv $file ${file//macosx_10_1[0-9]/macosx_10_11} || true; done

      - name: Upload wheels
        shell: bash
        run: |
          pip install awscli
          aws s3 sync --exact-timestamps ./bindings/python/dist "s3://tokenizers-releases/python/$DIST_DIR"

  create_wheels_macos_13:
    name: MacOS 13
    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        os: [macos-13]
        python: ["3.7", "3.8", "3.9", "3.10", "3.11"]
    steps:
      - name: Checkout repository
        uses: actions/checkout@v2

      - name: Install Rust
        uses: actions-rs/toolchain@v1
        with:
          toolchain: stable
          override: true

      - name: Install Python
        uses: actions/setup-python@v4
        with:
          python-version: ${{ matrix.python }}
          architecture: x64

      - name: Install dependencies
        run: |
          # On old versions of python there is an old version of setuptools already installed
          pip install setuptools wheel setuptools-rust==0.11.3 --ignore-installed --force-reinstall
      - name: Build wheel
        working-directory: ./bindings/python
        run: python setup.py bdist_wheel

      - name: Rename wheels
        shell: bash
        working-directory: ./bindings/python/dist
        run: for file in *.whl ; do mv $file ${file//macosx_10_1[0-9]/macosx_10_11} || true; done

      - name: Upload wheels
        shell: bash
        run: |
          pip install awscli
          aws s3 sync --exact-timestamps ./bindings/python/dist "s3://tokenizers-releases/python/$DIST_DIR"

  create_wheels_macos_arm64:
    name: Create wheels for MacOS M1
    runs-on: macos-arm64
    strategy:
      matrix:
        python: [ "3.9.13", "3.10.6", "3.11.1"]
        target: ["12.0", "13.0"]
    steps:
      - name: Checkout repository
        uses: actions/checkout@v3

      - name: Install Rust
        uses: actions-rs/toolchain@v1
        with:
          toolchain: stable
          override: true

      - name: Install Python
        shell: bash
        run: |
          echo $HOME
          export PYENV_ROOT="$HOME/.pyenv"
          command -v pyenv >/dev/null || export PATH="$PYENV_ROOT/bin:$PATH"
          eval "$(pyenv init -)"
          pyenv shell ${{ matrix.python }}
          which pyenv
          which python
          pip install -U setuptools wheel setuptools-rust awscli
          cd ./bindings/python
          MACOSX_DEPLOYMENT_TARGET=${{ matrix.target }} python setup.py bdist_wheel
          cd ../../
          aws s3 sync --exact-timestamps ./bindings/python/dist "s3://tokenizers-releases/python/$DIST_DIR"

  upload_package:
    name: Upload package to PyPi
    runs-on: ubuntu-latest
    needs: [create_wheels_manylinux, create_wheels_windows_32bit, create_wheels_others_64bit, create_wheels_macos_arm64, create_wheels_macos_13]

    steps:
      - uses: actions/checkout@v3

      - name: Install Python
        uses: actions/setup-python@v1

      - name: Retrieve all wheels
        shell: bash
        run: |
          pip install awscli
          aws s3 sync "s3://tokenizers-releases/python/$DIST_DIR" ./bindings/python/dist

      - name: Install dependencies
        run: |
          pip install setuptools wheel setuptools-rust

      - name: Create source distribution
        working-directory: ./bindings/python
        run: sh build-sdist.sh

      - name: Upload to PyPi
        working-directory: ./bindings/python
        run: |
          pip install twine
          twine upload dist/* -u __token__ -p "$PYPI_TOKEN"