mii/tokenizers
1
0
Fork 0
mirror of https://github.com/mii443/tokenizers.git synced 2025-08-23 16:49:27 +00:00
Code Issues Packages Projects Releases Wiki Activity

Python - Add Encoding length

Browse Source
This commit is contained in:
Anthony MOI
2020-02-18 18:24:13 -05:00
parent 0dc857ea8c
commit d8a73c89a7
1 changed files with 8 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
bindings/python/src/encoding.rs
View File

@ -4,7 +4,7 @@ use crate::error::PyError;
use pyo3::exceptions; use pyo3::exceptions;
use pyo3::prelude::*; use pyo3::prelude::*;
use pyo3::types::*; use pyo3::types::*;
use pyo3::{PyMappingProtocol, PyObjectProtocol}; use pyo3::{PyMappingProtocol, PyObjectProtocol, PySequenceProtocol};
use tk::tokenizer::PaddingDirection; use tk::tokenizer::PaddingDirection;
fn get_range(item: PyObject, max_len: usize) -> PyResult<std::ops::Range<usize>> { fn get_range(item: PyObject, max_len: usize) -> PyResult<std::ops::Range<usize>> {
@ -133,6 +133,13 @@ impl PyObjectProtocol for Encoding {
} }
} }
#[pyproto]
impl PySequenceProtocol for Encoding {
fn __len__(self) -> PyResult<usize> {
Ok(self.encoding.get_ids().len())
}
}
#[pymethods] #[pymethods]
impl Encoding { impl Encoding {
#[getter] #[getter]