mii/tokenizers
1
0
Fork 0
mirror of https://github.com/mii443/tokenizers.git synced 2025-08-22 16:25:30 +00:00
Code Issues Packages Projects Releases Wiki Activity

#[allow(dead_code)] // Suppress the "method is never used" warning

Browse Source
This commit is contained in:
Arthur Zucker
2023-09-05 16:42:22 +00:00
parent c3fa75fa0e
commit b57e1c3f5d
1 changed files with 7 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
tokenizers/src/tokenizer/added_vocabulary.rs
View File

@ -180,8 +180,8 @@ impl AddedVocabulary {
split_normalized_trie: (normalized_trie, vec![]),
}
}
/// Size of the additional vocabulary
#[allow(dead_code)] // Suppress the "method is never used" warning
pub fn len(&self) -> usize {
self.added_tokens_map.len()
}
@ -585,7 +585,9 @@ mod tests {
),
1
);
assert_eq!(vocab.len(), 1);
let vocab_len: usize = vocab.len();
assert_eq!(vocab_len, 1);
// Does not add multiple time the same token
assert_eq!(
@ -685,7 +687,7 @@ mod tests {
assert_eq!(token.content, "hey"); // Token was already there
token.special = true;
assert_eq!(token.special, true); // Token was already there
assert!(token.special); // Token was already there
}
#[test]
@ -821,6 +823,8 @@ mod tests {
let mut vocab = AddedVocabulary::new();
let normalizer = Lowercase;
assert_eq!(vocab.len(), 0);
vocab.add_tokens(
&[AddedToken::from("<mask>", false).single_word(true)],
&model,