Adressing first pass of comments.

2025-12-07 05:08:24 +00:00 · 2020-09-23 11:29:17 +02:00
parent 1cd4824273
commit 8f8156fd2c
10 changed files with 196 additions and 125 deletions
--- a/bindings/python/tests/implementations/test_byte_level_bpe.py
+++ b/bindings/python/tests/implementations/test_byte_level_bpe.py
@@ -67,7 +67,10 @@ class TestByteLevelBPE:

    def test_lowerspace(self, roberta_files):
        tokenizer = ByteLevelBPETokenizer.from_files(
-            roberta_files["vocab"], roberta_files["merges"], add_prefix_space=True, lowercase=True,
+            roberta_files["vocab"],
+            roberta_files["merges"],
+            add_prefix_space=True,
+            lowercase=True,
        )
        output = tokenizer.encode("The Quick Brown Fox Jumps Over The Lazy Dog")