mii/tokenizers
1
0
Fork 0
mirror of https://github.com/mii443/tokenizers.git synced 2025-12-05 04:08:22 +00:00
Code Issues Packages Projects Releases Wiki Activity

Removing forgotten places.

Browse Source
This commit is contained in:
Nicolas Patry
2020-08-22 10:14:53 +02:00
committed by Anthony MOI
parent 857948e5b8
commit 76b86f6901
3 changed files with 1 additions and 12 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
bindings/python/py_src/tokenizers/pre_tokenizers/__init__.py
View File

@@ -3,7 +3,6 @@ from .. import pre_tokenizers
PreTokenizer = pre_tokenizers.PreTokenizer
ByteLevel = pre_tokenizers.ByteLevel
Whitespace = pre_tokenizers.Whitespace
Deduplication = pre_tokenizers.Deduplication
Punctuation = pre_tokenizers.Punctuation
Sequence = pre_tokenizers.Sequence
WhitespaceSplit = pre_tokenizers.WhitespaceSplit