mii/tokenizers
1
0
Fork 0
mirror of https://github.com/mii443/tokenizers.git synced 2025-09-01 23:09:34 +00:00
Code Issues Packages Projects Releases Wiki Activity

Handle vocab size with added tokens

Browse Source
This commit is contained in:
Anthony MOI
2019-12-19 20:19:56 -05:00
parent b7040e0412
commit f2b9c30ad9
2 changed files with 8 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
bindings/python/src/tokenizer.rs
View File

@ -36,9 +36,8 @@ impl Tokenizer {
}
}
#[getter]
fn get_vocab_size(&self) -> usize {
self.tokenizer.get_vocab_size()
fn get_vocab_size(&self, with_added_tokens: bool) -> usize {
self.tokenizer.get_vocab_size(with_added_tokens)
}
fn with_model(&mut self, model: &mut Model) -> PyResult<()> {