Expose vocabulary size

This commit is contained in:
Anthony MOI
2019-12-10 16:20:31 -05:00
parent 6c294c60b0
commit b4b31d73cd
4 changed files with 19 additions and 0 deletions

View File

@ -30,6 +30,11 @@ impl Tokenizer {
}
}
#[getter]
fn get_vocab_size(&self) -> usize {
self.tokenizer.get_vocab_size()
}
fn with_model(&mut self, model: &mut Model) -> PyResult<()> {
if let Some(model) = model.model.to_pointer() {
self.tokenizer.with_model(model);