mii/tokenizers
1
0
Fork 0
mirror of https://github.com/mii443/tokenizers.git synced 2025-08-22 16:25:30 +00:00
Code Issues Packages Projects Releases Wiki Activity

Expand documentation of UnigramTrainer (#770)

Browse Source 
* Expand documentation of UnigramTrainer

* Put doc at the source

* Add signature

* make style

Co-authored-by: Anthony Moi <m.anthony.moi@gmail.com>
This commit is contained in:
Sylvain Gugger
2021-08-12 16:12:26 +02:00
committed by GitHub
parent da4c7b10e4
commit 6616e699f7
2 changed files with 39 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

25
bindings/python/py_src/tokenizers/trainers/__init__.pyi
View File

@ -72,9 +72,32 @@ class UnigramTrainer(Trainer):
if not seen in the training dataset.
If the strings contain more than one character, only the first one
is kept.
shrinking_factor (:obj:`float`):
The shrinking factor used at each step of the training to prune the
vocabulary.
unk_token (:obj:`str`):
The token used for out-of-vocabulary tokens.
max_piece_length (:obj:`int`):
The maximum length of a given token.
n_sub_iterations (:obj:`int`):
The number of iterations of the EM algorithm to perform before
pruning the vocabulary.
"""
def __init__(self, vocab_size=8000, show_progress=True, special_tokens=[]):
def __init__(
self,
vocab_size=8000,
show_progress=True,
special_tokens=[],
shrinking_factor=0.75,
unk_token=None,
max_piece_length=16,
n_sub_iterations=2,
):
pass
class WordLevelTrainer(Trainer):

16
bindings/python/src/trainers.rs
View File

@ -669,8 +669,22 @@ impl PyWordLevelTrainer {
/// if not seen in the training dataset.
/// If the strings contain more than one character, only the first one
/// is kept.
///
/// shrinking_factor (:obj:`float`):
/// The shrinking factor used at each step of the training to prune the
/// vocabulary.
///
/// unk_token (:obj:`str`):
/// The token used for out-of-vocabulary tokens.
///
/// max_piece_length (:obj:`int`):
/// The maximum length of a given token.
///
/// n_sub_iterations (:obj:`int`):
/// The number of iterations of the EM algorithm to perform before
/// pruning the vocabulary.
#[pyclass(extends=PyTrainer, module = "tokenizers.trainers", name=UnigramTrainer)]
#[text_signature = "(self, vocab_size=8000, show_progress=True, special_tokens= [])"]
#[text_signature = "(self, vocab_size=8000, show_progress=True, special_tokens=[], shrinking_factor=0.75, unk_token=None, max_piece_length=16, n_sub_iterations=2)"]
pub struct PyUnigramTrainer {}
#[pymethods]
impl PyUnigramTrainer {