mii/tokenizers
1
0
Fork 0
mirror of https://github.com/mii443/tokenizers.git synced 2025-08-23 08:45:38 +00:00
Code Issues Packages Projects Releases Wiki Activity

Python - Include correct packages and stubs

Browse Source
This commit is contained in:
Anthony MOI
2020-01-06 15:24:17 -05:00
parent 70a17ce330
commit 0e41e0b327
1 changed files with 18 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

19
bindings/python/setup.py
View File

@ -13,6 +13,23 @@ setup(
url="https://github.com/huggingface/tokenizers",
license="Apache License 2.0",
rust_extensions=[RustExtension("tokenizers.tokenizers", binding=Binding.PyO3)],
packages=["tokenizers"],
packages=[
"tokenizers",
"tokenizers.models",
"tokenizers.decoders",
"tokenizers.normalizers",
"tokenizers.pre_tokenizers",
"tokenizers.processors",
"tokenizers.trainers"
],
package_data = {
'tokenizers': [ 'py.typed', '__init__.pyi' ],
'tokenizers.models': [ 'py.typed', '__init__.pyi' ],
'tokenizers.decoders': [ 'py.typed', '__init__.pyi' ],
'tokenizers.normalizers': [ 'py.typed', '__init__.pyi' ],
'tokenizers.pre_tokenizers': [ 'py.typed', '__init__.pyi' ],
'tokenizers.processors': [ 'py.typed', '__init__.pyi' ],
'tokenizers.trainers': [ 'py.typed', '__init__.pyi' ],
},
zip_safe=False,
)