mii/tokenizers
1
0
Fork 0
mirror of https://github.com/mii443/tokenizers.git synced 2025-08-24 00:59:19 +00:00
Code Issues Packages Projects Releases Wiki Activity

Python - Add train_from_iterator to implementations

Browse Source
This commit is contained in:
Anthony MOI
2021-01-06 17:07:56 -05:00
committed by Anthony MOI
parent 817c5ad317
commit d94fa220b6
9 changed files with 166 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
bindings/python/tests/implementations/test_byte_level_bpe.py
View File

@ -89,3 +89,11 @@ class TestByteLevelBPE:
tokenizer = ByteLevelBPETokenizer.from_file(roberta_files["vocab"], roberta_files["merges"])
multiprocessing_with_parallelism(tokenizer, False)
multiprocessing_with_parallelism(tokenizer, True)
def test_train_from_iterator(self):
text = ["A first sentence", "Another sentence", "And a last one"]
tokenizer = ByteLevelBPETokenizer()
tokenizer.train_from_iterator(text, show_progress=False)
output = tokenizer.encode("A sentence")
assert output.tokens == ["A", "Ġsentence"]