Fixing test dependency.

This commit is contained in:
Nicolas Patry
2020-09-02 09:31:19 +02:00
committed by Anthony MOI
parent ee3860c029
commit 7b5c2b92c6
3 changed files with 26 additions and 15 deletions

View File

@ -59,6 +59,15 @@ def openai_files(data_dir):
}
@pytest.fixture(scope="session")
def train_files(data_dir):
return {
"wagahaiwa": download(
"https://storage.googleapis.com/tokenizers/unigram_wagahaiwa_nekodearu.txt"
),
}
def multiprocessing_with_parallelism(tokenizer, enabled: bool):
"""
This helper can be used to test that disabling parallelism avoids dead locks when the