diff --git a/bindings/python/src/tokenizer.rs b/bindings/python/src/tokenizer.rs index 0b90af37..dc264832 100644 --- a/bindings/python/src/tokenizer.rs +++ b/bindings/python/src/tokenizer.rs @@ -4,6 +4,7 @@ use pyo3::exceptions; use pyo3::prelude::*; use super::models::Model; +use super::pre_tokenizers::PreTokenizer; use super::token::Token; #[pyclass] @@ -21,7 +22,7 @@ impl Tokenizer { Ok(()) } else { Err(exceptions::Exception::py_err( - "The model is already being used in another Tokenizer", + "The Model is already being used in another Tokenizer", )) } } @@ -32,7 +33,18 @@ impl Tokenizer { Ok(()) } else { Err(exceptions::Exception::py_err( - "The model is already being used in another Tokenizer", + "The Model is already being used in another Tokenizer", + )) + } + } + + fn with_pre_tokenizer(&mut self, pretok: &mut PreTokenizer) -> PyResult<()> { + if let Some(pretok) = pretok.pretok.to_pointer() { + self.tokenizer.with_pre_tokenizer(pretok); + Ok(()) + } else { + Err(exceptions::Exception::py_err( + "The PreTokenizer is already being used in another Tokenizer", )) } }