mii/tokenizers
1
0
Fork 0
mirror of https://github.com/mii443/tokenizers.git synced 2025-08-22 16:25:30 +00:00
Code Issues Packages Projects Releases Wiki Activity

Python - change with_added_tokens as kwarg

Browse Source
This commit is contained in:
Anthony MOI
2019-12-25 22:22:35 -05:00
parent 905c1eb77e
commit 1879cb0bcb
1 changed files with 15 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

17
bindings/python/src/tokenizer.rs
View File

@ -36,8 +36,21 @@ impl Tokenizer {
} }
} }
fn get_vocab_size(&self, with_added_tokens: bool) -> usize { #[args(kwargs = "**")]
self.tokenizer.get_vocab_size(with_added_tokens) fn get_vocab_size(&self, kwargs: Option<&PyDict>) -> PyResult<usize> {
let mut with_added_tokens = true;
if let Some(kwargs) = kwargs {
for (key, value) in kwargs {
let key: &str = key.extract()?;
match key {
"with_added_tokens" => with_added_tokens = value.extract()?,
_ => println!("Ignored unknown kwarg option {}", key),
}
}
}
Ok(self.tokenizer.get_vocab_size(with_added_tokens))
} }
fn with_model(&mut self, model: &mut Model) -> PyResult<()> { fn with_model(&mut self, model: &mut Model) -> PyResult<()> {