mii/tokenizers
1
0
Fork 0
mirror of https://github.com/mii443/tokenizers.git synced 2025-12-07 13:18:31 +00:00
Code Issues Packages Projects Releases Wiki Activity

Use WhitespaceSplit for BPETokenizer

Browse Source
This commit is contained in:
Anthony MOI
2020-01-17 18:33:29 -05:00
parent fc601289eb
commit 395f605fd2
2 changed files with 3 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
bindings/python/tokenizers/implementations/bpe.py
View File

@@ -30,7 +30,7 @@ class BPETokenizer(BaseTokenizer):
NFKC.new(),
Lowercase.new()
])
tokenizer.pre_tokenizer = pre_tokenizers.Whitespace.new()
tokenizer.pre_tokenizer = pre_tokenizers.WhitespaceSplit.new()
tokenizer.decoder = decoders.BPEDecoder.new(suffix=suffix)
parameters = {