mii/tokenizers
1
0
Fork 0
mirror of https://github.com/mii443/tokenizers.git synced 2025-08-22 16:25:30 +00:00
Code Issues Packages Projects Releases Wiki Activity

Python - add doctype to length in implementations spm unigram (#943)

Browse Source
This commit is contained in:
dctelus
2022-03-08 05:59:07 -05:00
committed by GitHub
parent 4a8f5db067
commit 98249dfb0f
1 changed files with 3 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
bindings/python/py_src/tokenizers/implementations/sentencepiece_unigram.py
View File

@ -101,6 +101,9 @@ class SentencePieceUnigramTokenizer(BaseTokenizer):
A list of special tokens the model should know of. A list of special tokens the model should know of.
unk_token (:obj:`str`, `optional`): unk_token (:obj:`str`, `optional`):
The unknown token to be used by the model. The unknown token to be used by the model.
length (:obj:`int`, `optional`):
The total number of sequences in the iterator. This is used to
provide meaningful progress tracking
""" """
trainer = trainers.UnigramTrainer( trainer = trainers.UnigramTrainer(