mii/tokenizers
1
0
Fork 0
mirror of https://github.com/mii443/tokenizers.git synced 2025-08-23 00:35:35 +00:00
Code Issues Packages Projects Releases Wiki Activity

Python - Add support for custom Normalizer

Browse Source
This commit is contained in:
Anthony MOI
2020-09-18 14:57:29 -04:00
committed by Anthony MOI
parent 9245471dcd
commit 8d04b22278
3 changed files with 167 additions and 42 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
bindings/python/src/tokenizer.rs
View File

@ -844,7 +844,7 @@ impl PyTokenizer {
mod test {
use super::*;
use crate::models::PyModel;
use crate::normalizers::{PyNormalizer, PyNormalizerWrapper};
use crate::normalizers::{PyNormalizer, PyNormalizerTypeWrapper};
use std::sync::Arc;
use tempfile::NamedTempFile;
use tk::normalizers::{Lowercase, NFKC};
@ -854,7 +854,7 @@ mod test {
let mut tokenizer = Tokenizer::new(PyModel::new(Arc::new(
tk::models::bpe::BPE::default().into(),
)));
tokenizer.with_normalizer(PyNormalizer::new(PyNormalizerWrapper::Sequence(vec![
tokenizer.with_normalizer(PyNormalizer::new(PyNormalizerTypeWrapper::Sequence(vec![
Arc::new(NFKC.into()),
Arc::new(Lowercase.into()),
])));