mii/tokenizers
1
0
Fork 0
mirror of https://github.com/mii443/tokenizers.git synced 2025-08-23 00:35:35 +00:00
Code Issues Packages Projects Releases Wiki Activity

Python - Add NormalizedString + doc/typings

Browse Source
This commit is contained in:
Anthony MOI
2020-01-06 17:55:22 -05:00
parent 6de04bbaea
commit 0079a7a6b7
4 changed files with 93 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
bindings/python/src/encoding.rs
View File

@ -1,6 +1,7 @@
extern crate tokenizers as tk;
use crate::error::PyError;
use crate::normalized_string::NormalizedString;
use pyo3::prelude::*;
use pyo3::types::*;
use tk::tokenizer::PaddingDirection;
@ -20,13 +21,8 @@ impl Encoding {
#[pymethods]
impl Encoding {
#[getter]
fn get_original(&self) -> String {
self.encoding.get_normalized().get_original().to_owned()
}
#[getter]
fn get_normalized(&self) -> String {
self.encoding.get_normalized().get().to_owned()
fn get_normalized(&self) -> NormalizedString {
NormalizedString::new(self.encoding.get_normalized().clone())
}
#[args(kwargs = "**")]