mirror of
https://github.com/mii443/tokenizers.git
synced 2025-08-24 00:59:19 +00:00
Python - Set a PreTokenizer in a model
This commit is contained in:
@ -4,6 +4,7 @@ use pyo3::exceptions;
|
||||
use pyo3::prelude::*;
|
||||
|
||||
use super::models::Model;
|
||||
use super::pre_tokenizers::PreTokenizer;
|
||||
use super::token::Token;
|
||||
|
||||
#[pyclass]
|
||||
@ -21,7 +22,7 @@ impl Tokenizer {
|
||||
Ok(())
|
||||
} else {
|
||||
Err(exceptions::Exception::py_err(
|
||||
"The model is already being used in another Tokenizer",
|
||||
"The Model is already being used in another Tokenizer",
|
||||
))
|
||||
}
|
||||
}
|
||||
@ -32,7 +33,18 @@ impl Tokenizer {
|
||||
Ok(())
|
||||
} else {
|
||||
Err(exceptions::Exception::py_err(
|
||||
"The model is already being used in another Tokenizer",
|
||||
"The Model is already being used in another Tokenizer",
|
||||
))
|
||||
}
|
||||
}
|
||||
|
||||
fn with_pre_tokenizer(&mut self, pretok: &mut PreTokenizer) -> PyResult<()> {
|
||||
if let Some(pretok) = pretok.pretok.to_pointer() {
|
||||
self.tokenizer.with_pre_tokenizer(pretok);
|
||||
Ok(())
|
||||
} else {
|
||||
Err(exceptions::Exception::py_err(
|
||||
"The PreTokenizer is already being used in another Tokenizer",
|
||||
))
|
||||
}
|
||||
}
|
||||
|
Reference in New Issue
Block a user