mii/tokenizers
1
0
Fork 0
mirror of https://github.com/mii443/tokenizers.git synced 2025-08-22 16:25:30 +00:00
Code Issues Packages Projects Releases Wiki Activity

Python - Provide __repr__ for Encoding

Browse Source
This commit is contained in:
Anthony MOI
2020-01-07 21:33:45 -05:00
parent 05f683ce23
commit dc76e11768
1 changed files with 12 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
bindings/python/src/encoding.rs
View File

@ -4,6 +4,7 @@ use crate::error::PyError;
use crate::normalized_string::NormalizedString;
use pyo3::prelude::*;
use pyo3::types::*;
use pyo3::PyObjectProtocol;
use tk::tokenizer::PaddingDirection;
#[pyclass(dict)]
@ -18,6 +19,17 @@ impl Encoding {
}
}
#[pyproto]
impl PyObjectProtocol for Encoding {
fn __repr__(&self) -> PyResult<String> {
Ok(format!(
"Encoding(num_tokens={}, attributs=[ids, type_ids, tokens, offsets, \
attention_mask, special_tokens_mask, overflowing])",
self.encoding.get_ids().len()
))
}
}
#[pymethods]
impl Encoding {
#[getter]