mii/tokenizers
1
0
Fork 0
mirror of https://github.com/mii443/tokenizers.git synced 2025-08-23 00:35:35 +00:00
Code Issues Packages Projects Releases Wiki Activity

Python - PyNormalizer & PyPreTokenizer use a RwLock

Browse Source
This commit is contained in:
Anthony MOI
2020-11-16 11:37:28 -05:00
committed by Anthony MOI
parent 76d3b2128b
commit c22cfc31f9
3 changed files with 66 additions and 58 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
bindings/python/src/tokenizer.rs
View File

@ -1181,8 +1181,8 @@ mod test {
fn serialize() {
let mut tokenizer = Tokenizer::new(PyModel::from(BPE::default()));
tokenizer.with_normalizer(PyNormalizer::new(PyNormalizerTypeWrapper::Sequence(vec![
Arc::new(NFKC.into()),
Arc::new(Lowercase.into()),
Arc::new(RwLock::new(NFKC.into())),
Arc::new(RwLock::new(Lowercase.into())),
])));
let tmp = NamedTempFile::new().unwrap().into_temp_path();