Doc - Improve python and node tests

This commit is contained in:
Anthony MOI
2020-11-01 16:47:33 -05:00
committed by Anthony MOI
parent b6ffd9cba0
commit 324aa2930a
4 changed files with 106 additions and 47 deletions

View File

@@ -90,6 +90,14 @@ def doc_wiki_tokenizer(data_dir):
)
@pytest.fixture(scope="session")
def doc_pipeline_bert_tokenizer(data_dir):
return download(
"https://s3.amazonaws.com/models.huggingface.co/bert/anthony/doc-pipeline/tokenizer.json",
"bert-wiki.json",
)
def multiprocessing_with_parallelism(tokenizer, enabled: bool):
"""
This helper can be used to test that disabling parallelism avoids dead locks when the