mii/tokenizers
1
0
Fork 0
mirror of https://github.com/mii443/tokenizers.git synced 2025-08-23 00:35:35 +00:00
Code Issues Packages Projects Releases Wiki Activity

update the get_vocab_size to compute actual length of the get_vocab function

Browse Source
This commit is contained in:
Arthur Zucker
2023-09-05 15:19:50 +00:00
parent f1da83f358
commit 531b06f6db
1 changed files with 5 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
tokenizers/src/tokenizer/mod.rs
View File

@ -668,12 +668,11 @@ where
pub fn get_vocab_size(&self, with_added_tokens: bool) -> usize { pub fn get_vocab_size(&self, with_added_tokens: bool) -> usize {
// TODO ArthurZ THIS IS WRONG! We need to measure the length of the `set` because // TODO ArthurZ THIS IS WRONG! We need to measure the length of the `set` because
// now some tokens can be both in the added_tokens_encoder and in the vocab // now some tokens can be both in the added_tokens_encoder and in the vocab
self.model.get_vocab_size() if with_added_tokens {
+ if with_added_tokens { self.get_vocab(with_added_tokens).len()
self.added_vocabulary.len() } else {
} else { self.model.get_vocab_size()
0 }
}
} }
/// Converts a token in the corresponding id. /// Converts a token in the corresponding id.