mii/tokenizers
1
0
Fork 0
mirror of https://github.com/mii443/tokenizers.git synced 2025-08-23 16:49:27 +00:00
Code Issues Packages Projects Releases Wiki Activity

Python - Update PyNormalizer interface

Browse Source
This commit is contained in:
Anthony MOI
2020-09-18 16:42:52 -04:00
committed by Anthony MOI
parent bd8f25ee2c
commit b6e7a6e2f7
4 changed files with 29 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
bindings/python/src/normalizers.rs
View File

@ -5,7 +5,7 @@ use pyo3::prelude::*;
use pyo3::types::*;
use crate::error::ToPyResult;
use crate::utils::PyNormalizedStringRefMut;
use crate::utils::{PyNormalizedString, PyNormalizedStringRefMut};
use serde::ser::SerializeStruct;
use serde::{Deserialize, Deserializer, Serialize, Serializer};
use tk::normalizers::{
@ -105,6 +105,10 @@ impl PyNormalizer {
}
}
fn normalize(&self, normalized: &mut PyNormalizedString) -> PyResult<()> {
ToPyResult(self.normalizer.normalize(&mut normalized.normalized)).into()
}
fn normalize_str(&self, sequence: &str) -> PyResult<String> {
let mut normalized = NormalizedString::from(sequence);
ToPyResult(self.normalizer.normalize(&mut normalized)).into_py()?;