mii/tokenizers
1
0
Fork 0
mirror of https://github.com/mii443/tokenizers.git synced 2025-08-23 00:35:35 +00:00
Code Issues Packages Projects Releases Wiki Activity

Adding node bindings.

Browse Source 
- simplify normalizer.
- simplify python bindings.
This commit is contained in:
Nicolas Patry
2020-09-16 16:18:30 +02:00
parent 792d618006
commit aea22a4004
4 changed files with 21 additions and 12 deletions
Download Patch File Download Diff File Expand all files Collapse all files

13
bindings/python/src/normalizers.rs
View File

@ -8,13 +8,8 @@ use crate::error::ToPyResult;
use serde::ser::SerializeStruct;
use serde::{Deserialize, Serialize, Serializer};
use tk::normalizers::{
<<<<<<< HEAD
BertNormalizer, Lowercase, Nmt, NormalizerWrapper, Precompiled, Strip, StripAccents, NFC, NFD,
NFKC, NFKD,
=======
BertNormalizer, Lowercase, Nmt, NormalizerWrapper, Precompiled, Replace, Strip, NFC, NFD, NFKC,
NFKD,
>>>>>>> Adding a new "Replace" normalizer that takes a string and replaces it
BertNormalizer, Lowercase, Nmt, NormalizerWrapper, Precompiled, Replace, Strip, StripAccents,
NFC, NFD, NFKC, NFKD,
};
use tk::{NormalizedString, Normalizer};
use tokenizers as tk;
@ -335,9 +330,7 @@ pub struct PyReplace {}
#[pymethods]
impl PyReplace {
#[new]
fn new(pattern: &PyString, content: &PyString) -> PyResult<(Self, PyNormalizer)> {
let pattern: String = (*pattern).to_string()?.to_string();
let content: String = (*content).to_string()?.to_string();
fn new(pattern: String, content: String) -> PyResult<(Self, PyNormalizer)> {
Ok((PyReplace {}, Replace::new(pattern, content).into()))
}
}