Python - Set a PreTokenizer in a model

This commit is contained in:
Anthony MOI
2019-11-22 21:01:52 -05:00
parent 9b71c8f8de
commit e44f52024c

View File

@ -4,6 +4,7 @@ use pyo3::exceptions;
use pyo3::prelude::*;
use super::models::Model;
use super::pre_tokenizers::PreTokenizer;
use super::token::Token;
#[pyclass]
@ -21,7 +22,7 @@ impl Tokenizer {
Ok(())
} else {
Err(exceptions::Exception::py_err(
"The model is already being used in another Tokenizer",
"The Model is already being used in another Tokenizer",
))
}
}
@ -32,7 +33,18 @@ impl Tokenizer {
Ok(())
} else {
Err(exceptions::Exception::py_err(
"The model is already being used in another Tokenizer",
"The Model is already being used in another Tokenizer",
))
}
}
fn with_pre_tokenizer(&mut self, pretok: &mut PreTokenizer) -> PyResult<()> {
if let Some(pretok) = pretok.pretok.to_pointer() {
self.tokenizer.with_pre_tokenizer(pretok);
Ok(())
} else {
Err(exceptions::Exception::py_err(
"The PreTokenizer is already being used in another Tokenizer",
))
}
}