mirror of
https://github.com/mii443/tokenizers.git
synced 2025-08-22 16:25:30 +00:00
Python - Add WordPiece decoder
This commit is contained in:
@ -36,6 +36,18 @@ impl ByteLevel {
|
||||
}
|
||||
}
|
||||
|
||||
#[pyclass]
|
||||
pub struct WordPiece {}
|
||||
#[pymethods]
|
||||
impl WordPiece {
|
||||
#[staticmethod]
|
||||
fn new() -> PyResult<Decoder> {
|
||||
Ok(Decoder {
|
||||
decoder: Container::Owned(Box::new(tk::decoders::wordpiece::WordPiece)),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
struct PyDecoder {
|
||||
class: PyObject,
|
||||
}
|
||||
|
Reference in New Issue
Block a user