mii/tokenizers
1
0
Fork 0
mirror of https://github.com/mii443/tokenizers.git synced 2025-08-27 02:29:38 +00:00
Code Issues Packages Projects Releases Wiki Activity

Python - Tests for parallelism with multiprocessing

Browse Source 
Co-authored-by: Evan Pete Walsh <epwalsh10@gmail.com>
This commit is contained in:
Anthony MOI
2020-06-23 11:25:39 -04:00
parent 5f760df231
commit aa3b39f692
5 changed files with 55 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
bindings/python/tests/implementations/test_char_bpe.py
View File

@ -1,4 +1,4 @@
from ..utils import data_dir, openai_files
from ..utils import data_dir, openai_files, multiprocessing_with_parallelism
from tokenizers import CharBPETokenizer
@ -42,3 +42,8 @@ class TestBertWordPieceBPE:
tokenizer = CharBPETokenizer(openai_files["vocab"], openai_files["merges"], lowercase=True)
decoded = tokenizer.decode(tokenizer.encode("my name is john").ids)
assert decoded == "my name is john"
def test_multiprocessing_with_parallelism(self, openai_files):
tokenizer = CharBPETokenizer(openai_files["vocab"], openai_files["merges"])
multiprocessing_with_parallelism(tokenizer, False)
multiprocessing_with_parallelism(tokenizer, True)