mii/tokenizers
1
0
Fork 0
mirror of https://github.com/mii443/tokenizers.git synced 2025-09-01 14:59:20 +00:00
Code Issues Packages Projects Releases Wiki Activity

We use 19.10b0 not 20 here...

Browse Source
This commit is contained in:
Nicolas Patry
2020-09-23 11:58:15 +02:00
parent 4705fa8a00
commit 9672995a56
9 changed files with 16 additions and 50 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
bindings/python/tests/implementations/test_byte_level_bpe.py
View File

@ -67,10 +67,7 @@ class TestByteLevelBPE:
def test_lowerspace(self, roberta_files):
tokenizer = ByteLevelBPETokenizer.from_files(
roberta_files["vocab"],
roberta_files["merges"],
add_prefix_space=True,
lowercase=True,
roberta_files["vocab"], roberta_files["merges"], add_prefix_space=True, lowercase=True,
)
output = tokenizer.encode("The Quick Brown Fox Jumps Over The Lazy Dog")