mii/tokenizers
1
0
Fork 0
mirror of https://github.com/mii443/tokenizers.git synced 2025-08-23 16:49:27 +00:00
Code Issues Packages Projects Releases Wiki Activity

Doc - Fixes some CI fails

Browse Source
This commit is contained in:
Anthony MOI
2020-11-02 10:14:34 -05:00
committed by Anthony MOI
parent 324aa2930a
commit d788a950ac
4 changed files with 10 additions and 11 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
bindings/python/tests/documentation/test_pipeline.py
View File

@ -2,11 +2,12 @@ from ..utils import data_dir, doc_wiki_tokenizer, doc_pipeline_bert_tokenizer
from tokenizers import Tokenizer
def print(*args, **kwargs):
pass
class TestPipeline:
def test_pipeline(self, doc_wiki_tokenizer):
def print(*args, **kwargs):
pass
try:
# START reload_tokenizer
from tokenizers import Tokenizer
@ -142,7 +143,7 @@ class TestPipeline:
bert_tokenizer.save("data/bert-wiki.json")
# END bert_train_tokenizer
def test_bert_example(self):
def test_bert_example(self, doc_pipeline_bert_tokenizer):
try:
bert_tokenizer = Tokenizer.from_file("data/bert-wiki.json")
except Exception: