mii/tokenizers
1
0
Fork 0
mirror of https://github.com/mii443/tokenizers.git synced 2025-08-23 00:35:35 +00:00
Code Issues Packages Projects Releases Wiki Activity

Python - Test and fix classes pickling

Browse Source
This commit is contained in:
Anthony MOI
2020-05-18 19:52:10 -04:00
parent 6a70162d78
commit c5bba91bf4
11 changed files with 76 additions and 24 deletions
Download Patch File Download Diff File Expand all files Collapse all files

14
bindings/python/src/processors.rs
View File

@ -46,7 +46,7 @@ impl PostProcessor {
}
}
#[pyclass(extends=PostProcessor)]
#[pyclass(extends=PostProcessor, module = "tokenizers.processors")]
pub struct BertProcessing {}
#[pymethods]
impl BertProcessing {
@ -61,9 +61,13 @@ impl BertProcessing {
},
))
}
fn __getnewargs__<'p>(&self, py: Python<'p>) -> PyResult<&'p PyTuple> {
Ok(PyTuple::new(py, &[("", 0), ("", 0)]))
}
}
#[pyclass(extends=PostProcessor)]
#[pyclass(extends=PostProcessor, module = "tokenizers.processors")]
pub struct RobertaProcessing {}
#[pymethods]
impl RobertaProcessing {
@ -86,9 +90,13 @@ impl RobertaProcessing {
},
))
}
fn __getnewargs__<'p>(&self, py: Python<'p>) -> PyResult<&'p PyTuple> {
Ok(PyTuple::new(py, &[("", 0), ("", 0)]))
}
}
#[pyclass(extends=PostProcessor)]
#[pyclass(extends=PostProcessor, module = "tokenizers.processors")]
pub struct ByteLevel {}
#[pymethods]
impl ByteLevel {