mii/tokenizers
1
0
Fork 0
mirror of https://github.com/mii443/tokenizers.git synced 2025-08-22 16:25:30 +00:00
Code Issues Packages Projects Releases Wiki Activity

Python - Restore methods on Encoding

Browse Source
This commit is contained in:
Anthony MOI
2019-12-29 01:26:42 -05:00
parent 708a63514a
commit 049029dc42
1 changed files with 12 additions and 12 deletions
Download Patch File Download Diff File Expand all files Collapse all files

24
bindings/python/src/encoding.rs
View File

@ -16,15 +16,15 @@ impl Encoding {
#[pymethods]
impl Encoding {
// #[getter]
// fn get_original(&self) -> String {
// self.encoding.get_original().to_owned()
// }
#[getter]
fn get_original(&self) -> String {
self.encoding.get_normalized().get_original().to_owned()
}
// #[getter]
// fn get_normalized(&self) -> String {
// self.encoding.get_normalized().to_owned()
// }
#[getter]
fn get_normalized(&self) -> String {
self.encoding.get_normalized().get().to_owned()
}
#[getter]
fn get_ids(&self) -> Vec<u32> {
@ -41,10 +41,10 @@ impl Encoding {
self.encoding.get_type_ids().to_vec()
}
// #[getter]
// fn get_offsets(&self) -> Vec<(usize, usize)> {
// self.encoding.get_offsets().to_vec()
// }
#[getter]
fn get_offsets(&self) -> Vec<(usize, usize)> {
self.encoding.get_offsets().to_vec()
}
#[getter]
fn get_special_tokens_mask(&self) -> Vec<u32> {