Remove support for Python 3.5 (#714)

* Python - remove support for python 3.5

* revert ci

* revert build-wheels.sh

* Update CHANGELOG.md

.github/workflows/python-release-conda.yml

@@ -15,7 +15,7 @@ jobs:
       fail-fast: false
       matrix:
         os: [ubuntu-latest, windows-latest, macos-latest]
-        python: [3.5, 3.6, 3.7, 3.8, 3.9]
+        python: [3.6, 3.7, 3.8, 3.9]
     defaults:
       run:
         shell: bash -l {0}

.github/workflows/python-release.yml

@@ -30,7 +30,7 @@ jobs:
     runs-on: windows-latest
     strategy:
       matrix:
-        python: [3.5, 3.6, 3.7, 3.8, 3.9]
+        python: [3.6, 3.7, 3.8, 3.9]
     steps:
       - name: Checkout repository
         uses: actions/checkout@v1
@@ -76,7 +76,7 @@ jobs:
     strategy:
       matrix:
         os: [windows-latest, macos-10.15]
-        python: [3.5, 3.6, 3.7, 3.8, 3.9]
+        python: [3.6, 3.7, 3.8, 3.9]
     steps:
       - name: Checkout repository
         uses: actions/checkout@v1

.github/workflows/python.yml

@@ -16,7 +16,7 @@ jobs:
     runs-on: windows-latest
     strategy:
       matrix:
-        python: [3.5, 3.6, 3.7, 3.8]
+        python: [3.6, 3.7, 3.8, 3.9]
     steps:
       - name: Checkout repository
         uses: actions/checkout@v1

bindings/python/CHANGELOG.md

@@ -13,6 +13,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 ### Added
 - [#693]: Add a CTC Decoder for Wave2Vec models
 
+### Removed
+- [#714]: Removed support for Python 3.5
+
 ## [0.10.2]
 
 ### Fixed
@@ -318,6 +321,7 @@ delimiter (Works like `.split(delimiter)`)
 - Fix a bug that was causing crashes in Python 3.5
 
 
+[#714]: https://github.com/huggingface/tokenizers/pull/714
 [#707]: https://github.com/huggingface/tokenizers/pull/707
 [#693]: https://github.com/huggingface/tokenizers/pull/693
 [#686]: https://github.com/huggingface/tokenizers/pull/686

bindings/python/Makefile

@@ -7,12 +7,12 @@ dir_guard=@mkdir -p $(@D)
 # Format source code automatically
 style:
 	python stub.py
-	black --line-length 100 --target-version py35 examples py_src/tokenizers tests
+	black --line-length 100 --target-version py36 examples py_src/tokenizers tests
 
 # Check the source code is formatted correctly
 check-style:
 	python stub.py --check
-	black --check --line-length 100 --target-version py35 examples py_src/tokenizers tests
+	black --check --line-length 100 --target-version py36 examples py_src/tokenizers tests
 
 TESTS_RESOURCES = $(DATA_DIR)/small.txt $(DATA_DIR)/roberta.json
 

bindings/python/build-wheels.sh

@@ -4,7 +4,7 @@ set -ex
 curl https://sh.rustup.rs -sSf | sh -s -- -y
 export PATH="$HOME/.cargo/bin:$PATH"
 
-for PYBIN in /opt/python/{cp35-cp35m,cp36-cp36m,cp37-cp37m,cp38-cp38,cp39-cp39}/bin; do
+for PYBIN in /opt/python/{cp36-cp36m,cp37-cp37m,cp38-cp38,cp39-cp39}/bin; do
     export PYTHON_SYS_EXECUTABLE="$PYBIN/python"
 
     "${PYBIN}/pip" install -U setuptools-rust==0.11.3

bindings/python/pyproject.toml

@@ -3,5 +3,5 @@ requires = ["setuptools", "wheel", "setuptools-rust"]
 build-backend = "setuptools.build_meta"
 
 [tool.black]
-target-version = ['py35']
+target-version = ['py36']
 line-length = 100

bindings/python/stub.py

@@ -122,7 +122,7 @@ def py_file(module, origin):
 
 def do_black(content, is_pyi):
     mode = black.Mode(
-        target_versions={black.TargetVersion.PY35},
+        target_versions={black.TargetVersion.PY36},
         line_length=100,
         is_pyi=is_pyi,
         string_normalization=True,