mirror of
https://github.com/mii443/tokenizers.git
synced 2025-12-06 04:38:23 +00:00
Expose vocabulary size
This commit is contained in:
@@ -30,6 +30,11 @@ impl Tokenizer {
|
||||
}
|
||||
}
|
||||
|
||||
#[getter]
|
||||
fn get_vocab_size(&self) -> usize {
|
||||
self.tokenizer.get_vocab_size()
|
||||
}
|
||||
|
||||
fn with_model(&mut self, model: &mut Model) -> PyResult<()> {
|
||||
if let Some(model) = model.model.to_pointer() {
|
||||
self.tokenizer.with_model(model);
|
||||
|
||||
Reference in New Issue
Block a user