mii/tokenizers
1
0
Fork 0
mirror of https://github.com/mii443/tokenizers.git synced 2025-12-07 13:18:31 +00:00
Code Issues Packages Projects Releases Wiki Activity

Customize the doc for each language

Browse Source
This commit is contained in:
Anthony MOI
2020-10-01 16:53:04 -04:00
committed by Anthony MOI
parent 7366b9e797
commit e865b7cd7c
4 changed files with 44 additions and 31 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
bindings/python/tests/examples/test_load.py
View File

@@ -1,7 +1,8 @@
from tokenizers import Tokenizer
# START load
tokenizer = Tokenizer.from_file("data/roberta.json")
# END load
example = "This is an example"
ids = [713, 16, 41, 1246]