mii/tokenizers
1
0
Fork 0
mirror of https://github.com/mii443/tokenizers.git synced 2025-08-23 00:35:35 +00:00
Code Issues Packages Projects Releases Wiki Activity

Python - Adding tokenizers classes - WIP

Browse Source
This commit is contained in:
Anthony MOI
2020-01-07 16:20:20 -05:00
parent 6294d342d5
commit 63063118df
6 changed files with 220 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
bindings/python/setup.py
View File

@ -20,7 +20,8 @@ setup(
"tokenizers.normalizers",
"tokenizers.pre_tokenizers",
"tokenizers.processors",
"tokenizers.trainers"
"tokenizers.trainers",
"tokenizers.implementations",
],
package_data = {
'tokenizers': [ 'py.typed', '__init__.pyi' ],
@ -30,6 +31,7 @@ setup(
'tokenizers.pre_tokenizers': [ 'py.typed', '__init__.pyi' ],
'tokenizers.processors': [ 'py.typed', '__init__.pyi' ],
'tokenizers.trainers': [ 'py.typed', '__init__.pyi' ],
'tokenizers.implementations': [ 'py.typed' ],
},
zip_safe=False,
)