mii/tokenizers
1
0
Fork 0
mirror of https://github.com/mii443/tokenizers.git synced 2025-12-16 17:18:43 +00:00
Code Issues Packages Projects Releases Wiki Activity

typo: from_files() renamed to from_file()

Browse Source
This commit is contained in:
taufique74
2020-11-04 14:23:49 +06:00
committed by Anthony MOI
parent 20c7045ba1
commit 5cccaefcee
1 changed files with 1 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
bindings/python/tests/documentation/test_quicktour.py
View File

@@ -17,7 +17,7 @@ class TestQuicktour:
# END train # END train
# START reload_model # START reload_model
files = tokenizer.model.save("data", "wiki") files = tokenizer.model.save("data", "wiki")
tokenizer.model = BPE.from_files(*files, unk_token="[UNK]") tokenizer.model = BPE.from_file(*files, unk_token="[UNK]")
# END reload_model # END reload_model
# START save # START save
tokenizer.save("data/tokenizer-wiki.json") tokenizer.save("data/tokenizer-wiki.json")