mii/tokenizers
1
0
Fork 0
mirror of https://github.com/mii443/tokenizers.git synced 2025-12-16 17:18:43 +00:00
Code Issues Packages Projects Releases Wiki Activity

Rust - Fix WordLevelTrainer default values

Browse Source
This commit is contained in:
Anthony MOI
2020-12-08 13:16:03 -05:00
committed by Anthony MOI
parent d71e66e53c
commit a3a9561f9f
1 changed files with 4 additions and 10 deletions
Download Patch File Download Diff File Expand all files Collapse all files

14
tokenizers/src/models/wordlevel/trainer.rs
View File

@@ -7,13 +7,13 @@ use std::collections::HashMap;
#[derive(Debug, Clone, Builder)] #[derive(Debug, Clone, Builder)]
pub struct WordLevelTrainer { pub struct WordLevelTrainer {
/// The minimum frequency a word must have to be part of the vocabulary /// The minimum frequency a word must have to be part of the vocabulary
#[builder(default)] #[builder(default = "0")]
pub min_frequency: u32, pub min_frequency: u32,
/// The target vocabulary size /// The target vocabulary size
#[builder(default)] #[builder(default = "30_000")]
pub vocab_size: usize, pub vocab_size: usize,
/// Whether to show progress while training /// Whether to show progress while training
#[builder(default)] #[builder(default = "true")]
pub show_progress: bool, pub show_progress: bool,
/// A list of special tokens that the model should know of /// A list of special tokens that the model should know of
#[builder(default)] #[builder(default)]
@@ -25,13 +25,7 @@ pub struct WordLevelTrainer {
impl Default for WordLevelTrainer { impl Default for WordLevelTrainer {
fn default() -> Self { fn default() -> Self {
Self { Self::builder().build().unwrap()
min_frequency: 0,
vocab_size: 30_000,
show_progress: true,
special_tokens: vec![],
words: HashMap::new(),
}
} }
} }