mii/tokenizers
1
0
Fork 0
mirror of https://github.com/mii443/tokenizers.git synced 2025-08-23 00:35:35 +00:00
Code Issues Packages Projects Releases Wiki Activity

Handle vocab size with added tokens

Browse Source
This commit is contained in:
Anthony MOI
2019-12-19 20:19:56 -05:00
parent b7040e0412
commit f2b9c30ad9
2 changed files with 8 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
bindings/python/src/tokenizer.rs
View File

@ -36,9 +36,8 @@ impl Tokenizer {
} }
} }
#[getter] fn get_vocab_size(&self, with_added_tokens: bool) -> usize {
fn get_vocab_size(&self) -> usize { self.tokenizer.get_vocab_size(with_added_tokens)
self.tokenizer.get_vocab_size()
} }
fn with_model(&mut self, model: &mut Model) -> PyResult<()> { fn with_model(&mut self, model: &mut Model) -> PyResult<()> {

7
tokenizers/src/tokenizer/mod.rs
View File

@ -210,8 +210,13 @@ impl Tokenizer {
} }
/// Get the size of the vocabulary /// Get the size of the vocabulary
pub fn get_vocab_size(&self) -> usize { pub fn get_vocab_size(&self, with_added_tokens: bool) -> usize {
self.model.get_vocab_size() self.model.get_vocab_size()
+ if with_added_tokens {
self.added_tokens.len()
} else {
0
}
} }
/// Converts a token in the corresponding id. /// Converts a token in the corresponding id.