mii/tokenizers
1
0
Fork 0
mirror of https://github.com/mii443/tokenizers.git synced 2025-08-23 00:35:35 +00:00
Code Issues Packages Projects Releases Wiki Activity

Python - Test Models

Browse Source
This commit is contained in:
Anthony MOI
2020-04-01 14:45:46 -04:00
parent 53a7dbdaee
commit dbc23e20a9
4 changed files with 43 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
bindings/python/tests/utils.py
View File

@ -35,3 +35,12 @@ def roberta_files(data_dir):
"https://s3.amazonaws.com/models.huggingface.co/bert/roberta-base-merges.txt"
),
}
@pytest.fixture(scope="session")
def bert_files(data_dir):
return {
"vocab": download(
"https://s3.amazonaws.com/models.huggingface.co/bert/bert-base-uncased-vocab.txt"
),
}