From 23e7e42adf6234645c47e5fbaf61e85f7132bc98 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ga=C3=A9tan=20Lepage?= <33058747+GaetanLepage@users.noreply.github.com> Date: Tue, 27 May 2025 08:48:27 +0200 Subject: [PATCH] Fix data path in test_continuing_prefix_trainer_mismatch (#1747) --- bindings/python/tests/bindings/test_trainers.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/bindings/python/tests/bindings/test_trainers.py b/bindings/python/tests/bindings/test_trainers.py index 2e33b274..38b59944 100644 --- a/bindings/python/tests/bindings/test_trainers.py +++ b/bindings/python/tests/bindings/test_trainers.py @@ -14,7 +14,7 @@ from tokenizers import ( trainers, ) -from ..utils import data_dir, train_files +from ..utils import data_dir, train_files, DATA_PATH class TestBpeTrainer: @@ -287,7 +287,7 @@ class TestUnigram: trainer.initial_alphabet = ["d", "z"] assert sorted(trainer.initial_alphabet) == ["d", "z"] - def test_continuing_prefix_trainer_mismatch(self): + def test_continuing_prefix_trainer_mismatch(self, train_files): UNK = "[UNK]" special_tokens = [UNK] tokenizer = Tokenizer(models.BPE(unk_token=UNK, continuing_subword_prefix="##")) @@ -295,8 +295,9 @@ class TestUnigram: tokenizer.pre_tokenizer = pre_tokenizers.Sequence( [pre_tokenizers.Whitespace(), pre_tokenizers.Digits(individual_digits=True)] ) - tokenizer.train(files=["data/big.txt"], trainer=trainer) + tokenizer.train(files=[train_files["big"]], trainer=trainer) - tokenizer.save("data/tokenizer.json") + tokenizer_json = os.path.join(DATA_PATH, "tokenizer.json") + tokenizer.save(tokenizer_json) - tokenizer.from_file("data/tokenizer.json") + tokenizer.from_file(tokenizer_json)