mii/tokenizers
1
0
Fork 0
mirror of https://github.com/mii443/tokenizers.git synced 2025-08-23 16:49:27 +00:00
Code Issues Packages Projects Releases Wiki Activity

Python - Test CharBPETokenizer

Browse Source
This commit is contained in:
Anthony MOI
2020-04-01 15:12:34 -04:00
parent dbc23e20a9
commit 7fd7dfd113
2 changed files with 56 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
bindings/python/tests/utils.py
View File

@ -44,3 +44,15 @@ def bert_files(data_dir):
"https://s3.amazonaws.com/models.huggingface.co/bert/bert-base-uncased-vocab.txt"
),
}
@pytest.fixture(scope="session")
def openai_files(data_dir):
return {
"vocab": download(
"https://s3.amazonaws.com/models.huggingface.co/bert/openai-gpt-vocab.json"
),
"merges": download(
"https://s3.amazonaws.com/models.huggingface.co/bert/openai-gpt-merges.txt"
),
}