mii/tokenizers
1
0
Fork 0
mirror of https://github.com/mii443/tokenizers.git synced 2025-08-23 00:35:35 +00:00
Code Issues Packages Projects Releases Wiki Activity

PyModel uses a RwLock to allow modifications

Browse Source
This commit is contained in:
Anthony MOI
2020-10-08 19:33:30 -04:00
committed by Anthony MOI
parent 54c7210b2f
commit 284a1dbee7
10 changed files with 76 additions and 69 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
bindings/python/src/tokenizer.rs
View File

@ -1,5 +1,5 @@
use std::collections::HashMap;
use std::sync::Arc;
use std::sync::{Arc, RwLock};
use numpy::PyArray1;
use pyo3::exceptions;
@ -457,7 +457,8 @@ impl PyTokenizer {
}
fn __getnewargs__<'p>(&self, py: Python<'p>) -> PyResult<&'p PyTuple> {
let model: PyObject = PyModel::new(Arc::new(BPE::default().into())).into_py(py);
let model: PyObject =
PyModel::new(Arc::new(RwLock::new(BPE::default().into()))).into_py(py);
let args = PyTuple::new(py, vec![model]);
Ok(args)
}
@ -965,7 +966,7 @@ impl PyTokenizer {
/// Returns:
/// :obj:`Optional[str]`: An optional token, :obj:`None` if out of vocabulary
#[text_signature = "(self, id)"]
fn id_to_token(&self, id: u32) -> Option<&str> {
fn id_to_token(&self, id: u32) -> Option<String> {
self.tokenizer.id_to_token(id)
}