mii/tokenizers
1
0
Fork 0
mirror of https://github.com/mii443/tokenizers.git synced 2025-08-23 08:45:38 +00:00
Code Issues Packages Projects Releases Wiki Activity

Python - Tests for parallelism with multiprocessing

Browse Source 
Co-authored-by: Evan Pete Walsh <epwalsh10@gmail.com>
This commit is contained in:
Anthony MOI
2020-06-23 11:25:39 -04:00
parent 5f760df231
commit aa3b39f692
5 changed files with 55 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
bindings/python/tests/implementations/test_byte_level_bpe.py
View File

@ -1,4 +1,4 @@
from ..utils import data_dir, roberta_files
from ..utils import data_dir, roberta_files, multiprocessing_with_parallelism
from tokenizers import ByteLevelBPETokenizer
@ -79,3 +79,8 @@ class TestByteLevelBPE:
"Ġlazy",
"Ġdog",
]
def test_multiprocessing_with_parallelism(self, roberta_files):
tokenizer = ByteLevelBPETokenizer(roberta_files["vocab"], roberta_files["merges"])
multiprocessing_with_parallelism(tokenizer, False)
multiprocessing_with_parallelism(tokenizer, True)