mirror of
https://github.com/mii443/tokenizers.git
synced 2025-08-22 16:25:30 +00:00
Doc - Update Rust entities using roles
This commit is contained in:
@ -65,6 +65,10 @@ Main features:
|
|||||||
classmethod
|
classmethod
|
||||||
static method
|
static method
|
||||||
Tokenizer
|
Tokenizer
|
||||||
`Tokenizer <https://docs.rs/tokenizers/latest/tokenizers/tokenizer/struct.Tokenizer.html>`__
|
:rust:struct:`~tokenizers::tokenizer::Tokenizer`
|
||||||
Tokenizer.train
|
Tokenizer.train
|
||||||
`train <https://docs.rs/tokenizers/0.10.1/tokenizers/tokenizer/struct.Tokenizer.html#method.train>`__
|
:rust:meth:`~tokenizers::tokenizer::Tokenizer::train()`
|
||||||
|
Tokenizer.save
|
||||||
|
:rust:meth:`~tokenizers::tokenizer::Tokenizer::save()`
|
||||||
|
Tokenizer.from_file
|
||||||
|
:rust:meth:`~tokenizers::tokenizer::Tokenizer::from_file()`
|
||||||
|
@ -38,7 +38,7 @@ Training the tokenizer
|
|||||||
.. entities:: rust
|
.. entities:: rust
|
||||||
|
|
||||||
BpeTrainer
|
BpeTrainer
|
||||||
`BpeTrainer <https://docs.rs/tokenizers/latest/tokenizers/models/bpe/struct.BpeTrainer.html>`__
|
:rust:struct:`~tokenizers::models::bpe::BpeTrainer`
|
||||||
vocab_size
|
vocab_size
|
||||||
:obj:`vocab_size`
|
:obj:`vocab_size`
|
||||||
min_frequency
|
min_frequency
|
||||||
|
Reference in New Issue
Block a user