mii/tokenizers
1
0
Fork 0
mirror of https://github.com/mii443/tokenizers.git synced 2025-12-08 05:38:23 +00:00
Code Issues Packages Projects Releases Wiki Activity

Python - Tests for parallelism with multiprocessing

Browse Source 
Co-authored-by: Evan Pete Walsh <epwalsh10@gmail.com>
This commit is contained in:
Anthony MOI
2020-06-23 11:25:39 -04:00
parent 5f760df231
commit aa3b39f692
5 changed files with 55 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
bindings/python/tests/implementations/test_bert_wordpiece.py
View File

@@ -1,4 +1,4 @@
from ..utils import data_dir, bert_files
from ..utils import data_dir, bert_files, multiprocessing_with_parallelism
from tokenizers import BertWordPieceTokenizer
@@ -19,3 +19,8 @@ class TestBertWordPieceBPE:
assert output.tokens == ["my", "name", "is", "john", "pair"]
assert output.offsets == [(0, 2), (3, 7), (8, 10), (11, 15), (0, 4)]
assert output.type_ids == [0, 0, 0, 0, 1]
def test_multiprocessing_with_parallelism(self, bert_files):
tokenizer = BertWordPieceTokenizer(bert_files["vocab"])
multiprocessing_with_parallelism(tokenizer, False)
multiprocessing_with_parallelism(tokenizer, True)