mii/tokenizers
1
0
Fork 0
mirror of https://github.com/mii443/tokenizers.git synced 2025-09-01 23:09:34 +00:00
Code Issues Packages Projects Releases Wiki Activity

Doc - Hotfix training from iterators tutorial

Browse Source
This commit is contained in:
Anthony MOI
2021-02-03 15:50:09 -05:00
parent db22cb6315
commit 6a29dbc070
1 changed files with 1 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
bindings/python/tests/documentation/test_tutorial_train_from_iterators.py
View File

@ -14,7 +14,7 @@ class TestTrainFromIterators:
tokenizer = Tokenizer(models.Unigram()) tokenizer = Tokenizer(models.Unigram())
tokenizer.normalizer = normalizers.NFKC() tokenizer.normalizer = normalizers.NFKC()
tokenizer.pre_tokenizer = pre_tokenizers.ByteLevel() tokenizer.pre_tokenizer = pre_tokenizers.ByteLevel()
tokenizer.decoders = decoders.ByteLevel() tokenizer.decoder = decoders.ByteLevel()
trainer = trainers.UnigramTrainer( trainer = trainers.UnigramTrainer(
vocab_size=20000, vocab_size=20000,