mii/tokenizers
1
0
Fork 0
mirror of https://github.com/mii443/tokenizers.git synced 2025-09-02 07:19:24 +00:00
Code Issues Packages Projects Releases Wiki Activity

Black pre-commit after rebase.

Browse Source
This commit is contained in:
Nicolas Patry
2020-09-23 11:37:09 +02:00
parent acd4a7599f
commit 9b1ef9d895
8 changed files with 80 additions and 97 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
bindings/python/tests/implementations/test_char_bpe.py
View File

@ -6,9 +6,7 @@ from tokenizers import CharBPETokenizer
class TestBertWordPieceBPE:
def test_basic_encode(self, openai_files):
tokenizer = CharBPETokenizer.from_files(
openai_files["vocab"], openai_files["merges"]
)
tokenizer = CharBPETokenizer.from_files(openai_files["vocab"], openai_files["merges"])
output = tokenizer.encode("My name is John", "pair")
assert output.ids == [0, 253, 1362, 544, 0, 7, 12662, 2688]
@ -52,8 +50,6 @@ class TestBertWordPieceBPE:
assert decoded == "my name is john"
def test_multiprocessing_with_parallelism(self, openai_files):
tokenizer = CharBPETokenizer.from_files(
openai_files["vocab"], openai_files["merges"]
)
tokenizer = CharBPETokenizer.from_files(openai_files["vocab"], openai_files["merges"])
multiprocessing_with_parallelism(tokenizer, False)
multiprocessing_with_parallelism(tokenizer, True)