From 0d7c232f95a79ac72678e0cf66657dafb18f0451 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebastian=20P=C3=BCtz?= Date: Sat, 25 Jul 2020 19:41:18 +0200 Subject: [PATCH] Move Python source to subdirectory. This allows testing versions not built in-place. Otherwise importing (or testing) in the package root fails without develop builds. Replace maturin with setuptools_rust since maturin fails with proper project structure. --- .github/workflows/python.yml | 4 ++-- bindings/python/Makefile | 4 ++-- bindings/python/{ => py_src}/tokenizers/__init__.py | 0 bindings/python/{ => py_src}/tokenizers/__init__.pyi | 0 bindings/python/{ => py_src}/tokenizers/decoders/__init__.py | 0 bindings/python/{ => py_src}/tokenizers/decoders/__init__.pyi | 0 .../{ => py_src}/tokenizers/implementations/__init__.py | 0 .../{ => py_src}/tokenizers/implementations/base_tokenizer.py | 0 .../{ => py_src}/tokenizers/implementations/bert_wordpiece.py | 0 .../{ => py_src}/tokenizers/implementations/byte_level_bpe.py | 0 .../{ => py_src}/tokenizers/implementations/char_level_bpe.py | 0 .../tokenizers/implementations/sentencepiece_bpe.py | 0 bindings/python/{ => py_src}/tokenizers/models/__init__.py | 0 bindings/python/{ => py_src}/tokenizers/models/__init__.pyi | 0 .../python/{ => py_src}/tokenizers/normalizers/__init__.py | 0 .../python/{ => py_src}/tokenizers/normalizers/__init__.pyi | 0 .../python/{ => py_src}/tokenizers/pre_tokenizers/__init__.py | 0 .../{ => py_src}/tokenizers/pre_tokenizers/__init__.pyi | 0 .../python/{ => py_src}/tokenizers/processors/__init__.py | 0 .../python/{ => py_src}/tokenizers/processors/__init__.pyi | 0 bindings/python/{ => py_src}/tokenizers/trainers/__init__.py | 0 bindings/python/{ => py_src}/tokenizers/trainers/__init__.pyi | 0 bindings/python/setup.py | 1 + 23 files changed, 5 insertions(+), 4 deletions(-) rename bindings/python/{ => py_src}/tokenizers/__init__.py (100%) rename bindings/python/{ => py_src}/tokenizers/__init__.pyi (100%) rename bindings/python/{ => py_src}/tokenizers/decoders/__init__.py (100%) rename bindings/python/{ => py_src}/tokenizers/decoders/__init__.pyi (100%) rename bindings/python/{ => py_src}/tokenizers/implementations/__init__.py (100%) rename bindings/python/{ => py_src}/tokenizers/implementations/base_tokenizer.py (100%) rename bindings/python/{ => py_src}/tokenizers/implementations/bert_wordpiece.py (100%) rename bindings/python/{ => py_src}/tokenizers/implementations/byte_level_bpe.py (100%) rename bindings/python/{ => py_src}/tokenizers/implementations/char_level_bpe.py (100%) rename bindings/python/{ => py_src}/tokenizers/implementations/sentencepiece_bpe.py (100%) rename bindings/python/{ => py_src}/tokenizers/models/__init__.py (100%) rename bindings/python/{ => py_src}/tokenizers/models/__init__.pyi (100%) rename bindings/python/{ => py_src}/tokenizers/normalizers/__init__.py (100%) rename bindings/python/{ => py_src}/tokenizers/normalizers/__init__.pyi (100%) rename bindings/python/{ => py_src}/tokenizers/pre_tokenizers/__init__.py (100%) rename bindings/python/{ => py_src}/tokenizers/pre_tokenizers/__init__.pyi (100%) rename bindings/python/{ => py_src}/tokenizers/processors/__init__.py (100%) rename bindings/python/{ => py_src}/tokenizers/processors/__init__.pyi (100%) rename bindings/python/{ => py_src}/tokenizers/trainers/__init__.py (100%) rename bindings/python/{ => py_src}/tokenizers/trainers/__init__.pyi (100%) diff --git a/.github/workflows/python.yml b/.github/workflows/python.yml index 8b9a0a0d..feeb3404 100644 --- a/.github/workflows/python.yml +++ b/.github/workflows/python.yml @@ -114,6 +114,6 @@ jobs: run: | python -m venv .env source .env/bin/activate - pip install pytest requests maturin - maturin develop --release + pip install pytest requests setuptools_rust + python setup.py develop make test diff --git a/bindings/python/Makefile b/bindings/python/Makefile index a49a67f9..77834d74 100644 --- a/bindings/python/Makefile +++ b/bindings/python/Makefile @@ -2,11 +2,11 @@ # Format source code automatically style: - black --line-length 100 --target-version py35 examples tokenizers tests + black --line-length 100 --target-version py35 examples py_src/tokenizers tests # Check the source code is formatted correctly check-style: - black --check --line-length 100 --target-version py35 examples tokenizers tests + black --check --line-length 100 --target-version py35 examples py_src/tokenizers tests # Launch the test suite test: diff --git a/bindings/python/tokenizers/__init__.py b/bindings/python/py_src/tokenizers/__init__.py similarity index 100% rename from bindings/python/tokenizers/__init__.py rename to bindings/python/py_src/tokenizers/__init__.py diff --git a/bindings/python/tokenizers/__init__.pyi b/bindings/python/py_src/tokenizers/__init__.pyi similarity index 100% rename from bindings/python/tokenizers/__init__.pyi rename to bindings/python/py_src/tokenizers/__init__.pyi diff --git a/bindings/python/tokenizers/decoders/__init__.py b/bindings/python/py_src/tokenizers/decoders/__init__.py similarity index 100% rename from bindings/python/tokenizers/decoders/__init__.py rename to bindings/python/py_src/tokenizers/decoders/__init__.py diff --git a/bindings/python/tokenizers/decoders/__init__.pyi b/bindings/python/py_src/tokenizers/decoders/__init__.pyi similarity index 100% rename from bindings/python/tokenizers/decoders/__init__.pyi rename to bindings/python/py_src/tokenizers/decoders/__init__.pyi diff --git a/bindings/python/tokenizers/implementations/__init__.py b/bindings/python/py_src/tokenizers/implementations/__init__.py similarity index 100% rename from bindings/python/tokenizers/implementations/__init__.py rename to bindings/python/py_src/tokenizers/implementations/__init__.py diff --git a/bindings/python/tokenizers/implementations/base_tokenizer.py b/bindings/python/py_src/tokenizers/implementations/base_tokenizer.py similarity index 100% rename from bindings/python/tokenizers/implementations/base_tokenizer.py rename to bindings/python/py_src/tokenizers/implementations/base_tokenizer.py diff --git a/bindings/python/tokenizers/implementations/bert_wordpiece.py b/bindings/python/py_src/tokenizers/implementations/bert_wordpiece.py similarity index 100% rename from bindings/python/tokenizers/implementations/bert_wordpiece.py rename to bindings/python/py_src/tokenizers/implementations/bert_wordpiece.py diff --git a/bindings/python/tokenizers/implementations/byte_level_bpe.py b/bindings/python/py_src/tokenizers/implementations/byte_level_bpe.py similarity index 100% rename from bindings/python/tokenizers/implementations/byte_level_bpe.py rename to bindings/python/py_src/tokenizers/implementations/byte_level_bpe.py diff --git a/bindings/python/tokenizers/implementations/char_level_bpe.py b/bindings/python/py_src/tokenizers/implementations/char_level_bpe.py similarity index 100% rename from bindings/python/tokenizers/implementations/char_level_bpe.py rename to bindings/python/py_src/tokenizers/implementations/char_level_bpe.py diff --git a/bindings/python/tokenizers/implementations/sentencepiece_bpe.py b/bindings/python/py_src/tokenizers/implementations/sentencepiece_bpe.py similarity index 100% rename from bindings/python/tokenizers/implementations/sentencepiece_bpe.py rename to bindings/python/py_src/tokenizers/implementations/sentencepiece_bpe.py diff --git a/bindings/python/tokenizers/models/__init__.py b/bindings/python/py_src/tokenizers/models/__init__.py similarity index 100% rename from bindings/python/tokenizers/models/__init__.py rename to bindings/python/py_src/tokenizers/models/__init__.py diff --git a/bindings/python/tokenizers/models/__init__.pyi b/bindings/python/py_src/tokenizers/models/__init__.pyi similarity index 100% rename from bindings/python/tokenizers/models/__init__.pyi rename to bindings/python/py_src/tokenizers/models/__init__.pyi diff --git a/bindings/python/tokenizers/normalizers/__init__.py b/bindings/python/py_src/tokenizers/normalizers/__init__.py similarity index 100% rename from bindings/python/tokenizers/normalizers/__init__.py rename to bindings/python/py_src/tokenizers/normalizers/__init__.py diff --git a/bindings/python/tokenizers/normalizers/__init__.pyi b/bindings/python/py_src/tokenizers/normalizers/__init__.pyi similarity index 100% rename from bindings/python/tokenizers/normalizers/__init__.pyi rename to bindings/python/py_src/tokenizers/normalizers/__init__.pyi diff --git a/bindings/python/tokenizers/pre_tokenizers/__init__.py b/bindings/python/py_src/tokenizers/pre_tokenizers/__init__.py similarity index 100% rename from bindings/python/tokenizers/pre_tokenizers/__init__.py rename to bindings/python/py_src/tokenizers/pre_tokenizers/__init__.py diff --git a/bindings/python/tokenizers/pre_tokenizers/__init__.pyi b/bindings/python/py_src/tokenizers/pre_tokenizers/__init__.pyi similarity index 100% rename from bindings/python/tokenizers/pre_tokenizers/__init__.pyi rename to bindings/python/py_src/tokenizers/pre_tokenizers/__init__.pyi diff --git a/bindings/python/tokenizers/processors/__init__.py b/bindings/python/py_src/tokenizers/processors/__init__.py similarity index 100% rename from bindings/python/tokenizers/processors/__init__.py rename to bindings/python/py_src/tokenizers/processors/__init__.py diff --git a/bindings/python/tokenizers/processors/__init__.pyi b/bindings/python/py_src/tokenizers/processors/__init__.pyi similarity index 100% rename from bindings/python/tokenizers/processors/__init__.pyi rename to bindings/python/py_src/tokenizers/processors/__init__.pyi diff --git a/bindings/python/tokenizers/trainers/__init__.py b/bindings/python/py_src/tokenizers/trainers/__init__.py similarity index 100% rename from bindings/python/tokenizers/trainers/__init__.py rename to bindings/python/py_src/tokenizers/trainers/__init__.py diff --git a/bindings/python/tokenizers/trainers/__init__.pyi b/bindings/python/py_src/tokenizers/trainers/__init__.pyi similarity index 100% rename from bindings/python/tokenizers/trainers/__init__.pyi rename to bindings/python/py_src/tokenizers/trainers/__init__.pyi diff --git a/bindings/python/setup.py b/bindings/python/setup.py index 481f9c48..0bbf493d 100644 --- a/bindings/python/setup.py +++ b/bindings/python/setup.py @@ -31,6 +31,7 @@ setup( "Programming Language :: Python :: 3.8", "Topic :: Scientific/Engineering :: Artificial Intelligence", ], + package_dir={"": "py_src"}, packages=[ "tokenizers", "tokenizers.models",