mii/tokenizers
1
0
Fork 0
mirror of https://github.com/mii443/tokenizers.git synced 2025-08-23 00:35:35 +00:00
Code Issues Packages Projects Releases Wiki Activity

Python - Do not expose non working features of Encoding

Browse Source
This commit is contained in:
Anthony MOI
2019-12-17 17:43:42 -05:00
parent 0a3d4a86a9
commit 6766585965
1 changed files with 12 additions and 12 deletions
Download Patch File Download Diff File Expand all files Collapse all files

24
bindings/python/src/encoding.rs
View File

@ -27,15 +27,15 @@ impl PyObjectProtocol for Encoding {
#[pymethods]
impl Encoding {
#[getter]
fn get_original(&self) -> String {
self.encoding.get_original().to_owned()
}
// #[getter]
// fn get_original(&self) -> String {
// self.encoding.get_original().to_owned()
// }
#[getter]
fn get_normalized(&self) -> String {
self.encoding.get_normalized().to_owned()
}
// #[getter]
// fn get_normalized(&self) -> String {
// self.encoding.get_normalized().to_owned()
// }
#[getter]
fn get_ids(&self) -> Vec<u32> {
@ -52,10 +52,10 @@ impl Encoding {
self.encoding.get_type_ids().to_vec()
}
#[getter]
fn get_offsets(&self) -> Vec<(usize, usize)> {
self.encoding.get_offsets().to_vec()
}
// #[getter]
// fn get_offsets(&self) -> Vec<(usize, usize)> {
// self.encoding.get_offsets().to_vec()
// }
#[getter]
fn get_special_tokens_mask(&self) -> Vec<u32> {