Python - Add decoders

This commit is contained in:
Anthony MOI
2019-11-22 21:08:57 -05:00
parent e44f52024c
commit 8fbe3c2662
3 changed files with 44 additions and 2 deletions

View File

@ -0,0 +1,21 @@
extern crate tokenizers as tk;
use super::utils::Container;
use pyo3::prelude::*;
#[pyclass]
pub struct Decoder {
pub decoder: Container<dyn tk::tokenizer::Decoder + Sync>,
}
#[pyclass]
pub struct ByteLevel {}
#[pymethods]
impl ByteLevel {
#[staticmethod]
fn new() -> PyResult<Decoder> {
Ok(Decoder {
decoder: Container::Owned(Box::new(tk::decoders::byte_level::ByteLevel)),
})
}
}