mii/tokenizers
1
0
Fork 0
mirror of https://github.com/mii443/tokenizers.git synced 2025-08-22 16:25:30 +00:00
Code Issues Packages Projects Releases Wiki Activity

Switch from cached_download to hf_hub_download in tests (#1547)

Browse Source
This commit is contained in:
Lucain
2024-06-11 15:26:58 +02:00
committed by GitHub
parent 418c35c09e
commit 88f51fe7d2
1 changed files with 3 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
bindings/python/tests/test_serialization.py
View File

@ -3,9 +3,9 @@ import os
import unittest import unittest
import tqdm import tqdm
from huggingface_hub import hf_hub_download
from huggingface_hub import HfApi, cached_download, hf_hub_url
from tokenizers import Tokenizer from tokenizers import Tokenizer
from .utils import albert_base, data_dir from .utils import albert_base, data_dir
@ -63,7 +63,7 @@ class TestFullDeserialization(unittest.TestCase):
all_models = [("HueyNemud/das22-10-camembert_pretrained", "tokenizer.json")] all_models = [("HueyNemud/das22-10-camembert_pretrained", "tokenizer.json")]
for model_id, filename in tqdm.tqdm(all_models): for model_id, filename in tqdm.tqdm(all_models):
tokenizer_file = cached_download(hf_hub_url(model_id, filename=filename)) tokenizer_file = hf_hub_download(model_id, filename=filename)
is_ok = check(tokenizer_file) is_ok = check(tokenizer_file)
if not is_ok: if not is_ok: