mii/tokenizers
1
0
Fork 0
mirror of https://github.com/mii443/tokenizers.git synced 2025-08-23 00:35:35 +00:00
Code Issues Packages Projects Releases Wiki Activity

Python - PyModel uses a RwLock to allow modifications

Browse Source
This commit is contained in:
Anthony MOI
2020-11-13 16:17:20 -05:00
committed by Anthony MOI
parent dd399d2ad0
commit 7f3cfebf45
2 changed files with 21 additions and 34 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
bindings/python/src/tokenizer.rs
View File

@ -1,5 +1,4 @@
use std::collections::HashMap;
use std::sync::{Arc, RwLock};
use numpy::PyArray1;
use pyo3::exceptions;
@ -457,8 +456,7 @@ impl PyTokenizer {
}
fn __getnewargs__<'p>(&self, py: Python<'p>) -> PyResult<&'p PyTuple> {
let model: PyObject =
PyModel::new(Arc::new(RwLock::new(BPE::default().into()))).into_py(py);
let model = PyModel::from(BPE::default()).into_py(py);
let args = PyTuple::new(py, vec![model]);
Ok(args)
}
@ -1181,9 +1179,7 @@ mod test {
#[test]
fn serialize() {
let mut tokenizer = Tokenizer::new(PyModel::new(Arc::new(RwLock::new(
tk::models::bpe::BPE::default().into(),
))));
let mut tokenizer = Tokenizer::new(PyModel::from(BPE::default()));
tokenizer.with_normalizer(PyNormalizer::new(PyNormalizerTypeWrapper::Sequence(vec![
Arc::new(NFKC.into()),
Arc::new(Lowercase.into()),