mirror of
https://github.com/mii443/tokenizers.git
synced 2025-09-04 00:09:34 +00:00
Python - Add decoders
This commit is contained in:
21
bindings/python/src/decoders.rs
Normal file
21
bindings/python/src/decoders.rs
Normal file
@ -0,0 +1,21 @@
|
||||
extern crate tokenizers as tk;
|
||||
|
||||
use super::utils::Container;
|
||||
use pyo3::prelude::*;
|
||||
|
||||
#[pyclass]
|
||||
pub struct Decoder {
|
||||
pub decoder: Container<dyn tk::tokenizer::Decoder + Sync>,
|
||||
}
|
||||
|
||||
#[pyclass]
|
||||
pub struct ByteLevel {}
|
||||
#[pymethods]
|
||||
impl ByteLevel {
|
||||
#[staticmethod]
|
||||
fn new() -> PyResult<Decoder> {
|
||||
Ok(Decoder {
|
||||
decoder: Container::Owned(Box::new(tk::decoders::byte_level::ByteLevel)),
|
||||
})
|
||||
}
|
||||
}
|
Reference in New Issue
Block a user