Move Python source to subdirectory.

This allows testing versions not built in-place. Otherwise importing (or testing) in the package root fails without develop builds. Replace maturin with setuptools_rust since maturin fails with proper project structure.
2025-12-07 05:08:24 +00:00 · 2020-07-25 19:41:18 +02:00
parent e9a2e63a67
commit 0d7c232f95
23 changed files with 5 additions and 4 deletions
--- a/bindings/python/Makefile
+++ b/bindings/python/Makefile
@@ -2,11 +2,11 @@

 # Format source code automatically
 style:
-	black --line-length 100 --target-version py35 examples tokenizers tests
+	black --line-length 100 --target-version py35 examples py_src/tokenizers tests

 # Check the source code is formatted correctly
 check-style:
-	black --check --line-length 100 --target-version py35 examples tokenizers tests
+	black --check --line-length 100 --target-version py35 examples py_src/tokenizers tests

 # Launch the test suite
 test:
--- a/bindings/python/py_src/tokenizers/init.py
+++ b/bindings/python/py_src/tokenizers/init.py
--- a/bindings/python/py_src/tokenizers/init.pyi
+++ b/bindings/python/py_src/tokenizers/init.pyi
--- a/bindings/python/py_src/tokenizers/decoders/init.py
+++ b/bindings/python/py_src/tokenizers/decoders/init.py
--- a/bindings/python/py_src/tokenizers/decoders/init.pyi
+++ b/bindings/python/py_src/tokenizers/decoders/init.pyi
--- a/bindings/python/py_src/tokenizers/implementations/init.py
+++ b/bindings/python/py_src/tokenizers/implementations/init.py
--- a/bindings/python/py_src/tokenizers/implementations/base_tokenizer.py
+++ b/bindings/python/py_src/tokenizers/implementations/base_tokenizer.py
--- a/bindings/python/py_src/tokenizers/implementations/bert_wordpiece.py
+++ b/bindings/python/py_src/tokenizers/implementations/bert_wordpiece.py
--- a/bindings/python/py_src/tokenizers/implementations/byte_level_bpe.py
+++ b/bindings/python/py_src/tokenizers/implementations/byte_level_bpe.py
--- a/bindings/python/py_src/tokenizers/implementations/char_level_bpe.py
+++ b/bindings/python/py_src/tokenizers/implementations/char_level_bpe.py
--- a/bindings/python/py_src/tokenizers/implementations/sentencepiece_bpe.py
+++ b/bindings/python/py_src/tokenizers/implementations/sentencepiece_bpe.py
--- a/bindings/python/py_src/tokenizers/models/init.py
+++ b/bindings/python/py_src/tokenizers/models/init.py
--- a/bindings/python/py_src/tokenizers/models/init.pyi
+++ b/bindings/python/py_src/tokenizers/models/init.pyi
--- a/bindings/python/py_src/tokenizers/normalizers/init.py
+++ b/bindings/python/py_src/tokenizers/normalizers/init.py
--- a/bindings/python/py_src/tokenizers/normalizers/init.pyi
+++ b/bindings/python/py_src/tokenizers/normalizers/init.pyi
--- a/bindings/python/py_src/tokenizers/pre_tokenizers/init.py
+++ b/bindings/python/py_src/tokenizers/pre_tokenizers/init.py
--- a/bindings/python/py_src/tokenizers/pre_tokenizers/init.pyi
+++ b/bindings/python/py_src/tokenizers/pre_tokenizers/init.pyi
--- a/bindings/python/py_src/tokenizers/processors/init.py
+++ b/bindings/python/py_src/tokenizers/processors/init.py
--- a/bindings/python/py_src/tokenizers/processors/init.pyi
+++ b/bindings/python/py_src/tokenizers/processors/init.pyi
--- a/bindings/python/py_src/tokenizers/trainers/init.py
+++ b/bindings/python/py_src/tokenizers/trainers/init.py
--- a/bindings/python/py_src/tokenizers/trainers/init.pyi
+++ b/bindings/python/py_src/tokenizers/trainers/init.pyi
--- a/bindings/python/setup.py
+++ b/bindings/python/setup.py
@@ -31,6 +31,7 @@ setup(
        "Programming Language :: Python :: 3.8",
        "Topic :: Scientific/Engineering :: Artificial Intelligence",
    ],
+    package_dir={"": "py_src"},
    packages=[
        "tokenizers",
        "tokenizers.models",