mii/tokenizers
1
0
Fork 0
mirror of https://github.com/mii443/tokenizers.git synced 2025-08-23 16:49:27 +00:00
Code Issues Packages Projects Releases Wiki Activity

Remove Container from Normalizers, replace with Arc.

Browse Source 
* prefix the Python types in Rust with Py
* remove unsound Container wrappers, replace with Arc
This commit is contained in:
Sebastian Pütz
2020-07-25 15:44:28 +02:00
committed by Anthony MOI
parent 83a52c8080
commit 08b8c48127
3 changed files with 120 additions and 143 deletions
Download Patch File Download Diff File Expand all files Collapse all files

24
bindings/python/src/tokenizer.rs
View File

@ -16,7 +16,7 @@ use super::decoders::Decoder;
use super::encoding::PyEncoding;
use super::error::{PyError, ToPyResult};
use super::models::PyModel;
use super::normalizers::Normalizer;
use super::normalizers::PyNormalizer;
use super::pre_tokenizers::PreTokenizer;
use super::processors::PostProcessor;
use super::trainers::PyTrainer;
@ -268,7 +268,7 @@ impl From<PreTokenizedEncodeInput> for tk::tokenizer::EncodeInput {
}
}
type TokenizerImpl = Tokenizer<PyModel>;
type TokenizerImpl = Tokenizer<PyModel, PyNormalizer>;
#[pyclass(dict, module = "tokenizers")]
pub struct PyTokenizer {
@ -707,25 +707,13 @@ impl PyTokenizer {
}
#[getter]
fn get_normalizer(&self) -> PyResult<Option<Normalizer>> {
Ok(self
.tokenizer
.get_normalizer()
.map(|normalizer| Normalizer {
normalizer: Container::from_ref(normalizer),
}))
fn get_normalizer(&self) -> Option<PyNormalizer> {
self.tokenizer.get_normalizer().cloned()
}
#[setter]
fn set_normalizer(&mut self, mut normalizer: PyRefMut<Normalizer>) -> PyResult<()> {
if let Some(normalizer) = normalizer.normalizer.to_pointer() {
self.tokenizer.with_normalizer(normalizer);
Ok(())
} else {
Err(exceptions::Exception::py_err(
"The Normalizer is already being used in another Tokenizer",
))
}
fn set_normalizer(&mut self, normalizer: PyRef<PyNormalizer>) {
self.tokenizer.with_normalizer(normalizer.clone());
}
#[getter]