Python release CI (#2)

.github/workflows/python-release.yml

@@ -0,0 +1,92 @@
+name: Python Release
+
+on:
+  release:
+    types: [created]
+
+env:
+  AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+  AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+  AWS_DEFAULT_REGION: us-east-1
+  PYPI_TOKEN: ${{ secrets.PYPI_TOKEN_DIST }}
+
+jobs:
+
+  create_wheels_manylinux:
+    runs-on: ubuntu-latest
+    name: Create wheels for manylinux
+    container: quay.io/pypa/manylinux1_x86_64
+    steps:
+      - uses: actions/checkout@v1
+
+      - name: Build and audit wheels
+        working-directory: ./bindings/python
+        run: sh build-wheels.sh
+
+  create_wheels_others:
+    name: Create wheels for other OSes
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        os: [windows-latest, macos-latest]
+        python: [3.5, 3.6, 3.7, 3.8]
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v1
+
+      - name: Install Rust
+        uses: actions-rs/toolchain@v1
+        with:
+          toolchain: nightly-2019-11-01
+          override: true
+
+      - name: Install Python
+        uses: actions/setup-python@v1
+        with:
+          python-version: ${{ matrix.python }}
+          architecture: 'x64'
+
+      - name: Install dependencies
+        run: |
+          pip install setuptools wheel setuptools-rust
+
+      - name: Build wheel
+        working-directory: ./bindings/python
+        run: python setup.py bdist_wheel
+
+      - name: Upload wheels
+        shell: bash
+        run: |
+          pip install awscli
+          aws s3 sync --exact-timestamps ./bindings/python/dist "s3://tokenizers-releases/python/$GITHUB_SHA"
+
+  upload_package:
+    name: Upload package to PyPi
+    runs-on: ubuntu-latest
+    needs: [create_wheels_manylinux, create_wheels_others]
+
+    steps:
+      - uses: actions/checkout@v1
+
+      - name: Install Python
+        uses: actions/setup-python@v1
+
+      - name: Retrieve all wheels
+        shell: bash
+        run: |
+          pip install awscli
+          aws s3 sync "s3://tokenizers-releases/python/$GITHUB_SHA" ./bindings/python/dist
+
+      - name: Install dependencies
+        run: |
+          pip install setuptools wheel setuptools-rust
+
+      - name: Create source distribution
+        working-directory: ./bindings/python
+        run: sh build-sdist.sh
+
+      - name: Upload to PyPi
+        working-directory: ./bindings/python
+        run: |
+          pip install twine
+          twine upload dist/* -u __token__ -p "$PYPI_TOKEN"

bindings/python/MANIFEST.in

@@ -0,0 +1,4 @@
+include Cargo.toml
+recursive-include src *
+recursive-include tokenizers-lib *
+recursive-exclude tokenizers-lib/target *

bindings/python/build-sdist.sh

@@ -0,0 +1,9 @@
+#!/bin/bash
+set -ex
+
+# Create a symlink for tokenizers-lib
+ln -sf ../../tokenizers tokenizers-lib
+# Modify cargo.toml to include this symlink
+sed -i 's/\.\.\/\.\.\/tokenizers/\.\/tokenizers-lib/' Cargo.toml
+# Build the source distribution
+python setup.py sdist

bindings/python/build-wheels.sh

@@ -4,9 +4,7 @@ set -ex
 curl https://sh.rustup.rs -sSf | sh -s -- --default-toolchain nightly-2019-11-01 -y
 export PATH="$HOME/.cargo/bin:$PATH"
 
-cd /io/bindings/python
-
-for PYBIN in /opt/python/{cp35-cp35m,cp36-cp36m,cp37-cp37m}/bin; do
+for PYBIN in /opt/python/{cp35-cp35m,cp36-cp36m,cp37-cp37m,cp38-cp38}/bin; do
     export PYTHON_SYS_EXECUTABLE="$PYBIN/python"
 
     "${PYBIN}/pip" install -U setuptools-rust
@@ -16,3 +14,10 @@ done
 for whl in dist/*.whl; do
     auditwheel repair "$whl" -w dist/
 done
+
+# Keep only manylinux wheels
+rm dist/*-linux_*
+
+# Upload wheels
+/opt/python/cp37-cp37m/bin/pip install -U awscli
+/opt/python/cp37-cp37m/bin/python -m awscli s3 sync --exact-timestamps ./dist "s3://tokenizers-releases/python/$GITHUB_SHA"

bindings/python/setup.py

@@ -5,6 +5,9 @@ setup(
     name="tokenizers",
     version="0.0.2",
     description="Fast and Customizable Tokenizers",
+    long_description=open("README.md", "r", encoding="utf-8").read(),
+    long_description_content_type="text/markdown",
+    keywords="NLP tokenizer BPE transformer deep learning"
     author="Anthony MOI",
     author_email="anthony@huggingface.co",
     url="https://github.com/huggingface/tokenizers",