mii/tokenizers
1
0
Fork 0
mirror of https://github.com/mii443/tokenizers.git synced 2025-08-22 16:25:30 +00:00
Code Issues Packages Projects Releases Wiki Activity

#[allow(dead_code)] // Suppress the "method is never used" warning

Browse Source
This commit is contained in:
Arthur Zucker
2023-09-05 16:42:22 +00:00
parent c3fa75fa0e
commit b57e1c3f5d
1 changed files with 7 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
tokenizers/src/tokenizer/added_vocabulary.rs
View File

@ -180,8 +180,8 @@ impl AddedVocabulary {
split_normalized_trie: (normalized_trie, vec![]), split_normalized_trie: (normalized_trie, vec![]),
} }
} }
/// Size of the additional vocabulary /// Size of the additional vocabulary
#[allow(dead_code)] // Suppress the "method is never used" warning
pub fn len(&self) -> usize { pub fn len(&self) -> usize {
self.added_tokens_map.len() self.added_tokens_map.len()
} }
@ -585,7 +585,9 @@ mod tests {
), ),
1 1
); );
assert_eq!(vocab.len(), 1);
let vocab_len: usize = vocab.len();
assert_eq!(vocab_len, 1);
// Does not add multiple time the same token // Does not add multiple time the same token
assert_eq!( assert_eq!(
@ -685,7 +687,7 @@ mod tests {
assert_eq!(token.content, "hey"); // Token was already there assert_eq!(token.content, "hey"); // Token was already there
token.special = true; token.special = true;
assert_eq!(token.special, true); // Token was already there assert!(token.special); // Token was already there
} }
#[test] #[test]
@ -821,6 +823,8 @@ mod tests {
let mut vocab = AddedVocabulary::new(); let mut vocab = AddedVocabulary::new();
let normalizer = Lowercase; let normalizer = Lowercase;
assert_eq!(vocab.len(), 0);
vocab.add_tokens( vocab.add_tokens(
&[AddedToken::from("<mask>", false).single_word(true)], &[AddedToken::from("<mask>", false).single_word(true)],
&model, &model,