mirror of
https://github.com/mii443/tokenizers.git
synced 2025-08-23 00:35:35 +00:00
Python - Add ByteLevel PostProcessor
This commit is contained in:
@ -42,3 +42,17 @@ impl RobertaProcessing {
|
||||
}))
|
||||
}
|
||||
}
|
||||
|
||||
#[pyclass(extends=PostProcessor)]
|
||||
pub struct ByteLevel {}
|
||||
#[pymethods]
|
||||
impl ByteLevel {
|
||||
#[new]
|
||||
fn new(obj: &PyRawObject) -> PyResult<()> {
|
||||
Ok(obj.init(PostProcessor {
|
||||
processor: Container::Owned(Box::new(tk::processors::byte_level::ByteLevel::new(
|
||||
false,
|
||||
))),
|
||||
}))
|
||||
}
|
||||
}
|
||||
|
Reference in New Issue
Block a user