mii/tokenizers
1
0
Fork 0
mirror of https://github.com/mii443/tokenizers.git synced 2025-12-05 20:28:22 +00:00
Code Issues Packages Projects Releases Wiki Activity

Python - Set a PreTokenizer in a model

Browse Source
This commit is contained in:
Anthony MOI
2019-11-22 21:01:52 -05:00
parent 9b71c8f8de
commit e44f52024c
1 changed files with 14 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

16
bindings/python/src/tokenizer.rs
View File

@@ -4,6 +4,7 @@ use pyo3::exceptions;
use pyo3::prelude::*;
use super::models::Model;
use super::pre_tokenizers::PreTokenizer;
use super::token::Token;
#[pyclass]
@@ -21,7 +22,7 @@ impl Tokenizer {
Ok(())
} else {
Err(exceptions::Exception::py_err(
"The model is already being used in another Tokenizer",
"The Model is already being used in another Tokenizer",
))
}
}
@@ -32,7 +33,18 @@ impl Tokenizer {
Ok(())
} else {
Err(exceptions::Exception::py_err(
"The model is already being used in another Tokenizer",
"The Model is already being used in another Tokenizer",
))
}
}
fn with_pre_tokenizer(&mut self, pretok: &mut PreTokenizer) -> PyResult<()> {
if let Some(pretok) = pretok.pretok.to_pointer() {
self.tokenizer.with_pre_tokenizer(pretok);
Ok(())
} else {
Err(exceptions::Exception::py_err(
"The PreTokenizer is already being used in another Tokenizer",
))
}
}