mii/tokenizers
1
0
Fork 0
mirror of https://github.com/mii443/tokenizers.git synced 2025-09-01 23:09:34 +00:00
Code Issues Packages Projects Releases Wiki Activity

Python - Update some naming

Browse Source
This commit is contained in:
Anthony MOI
2020-01-08 09:54:03 -05:00
parent 45c6382c35
commit fc56f8d186
2 changed files with 18 additions and 18 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
bindings/python/src/tokenizer.rs
View File

@ -56,7 +56,7 @@ impl Tokenizer {
}
#[args(kwargs = "**")]
fn with_truncation(&mut self, max_length: usize, kwargs: Option<&PyDict>) -> PyResult<()> {
fn enable_truncation(&mut self, max_length: usize, kwargs: Option<&PyDict>) -> PyResult<()> {
let mut stride = 0;
let mut strategy = TruncationStrategy::LongestFirst;
@ -93,12 +93,12 @@ impl Tokenizer {
Ok(())
}
fn without_truncation(&mut self) {
fn no_truncation(&mut self) {
self.tokenizer.with_truncation(None);
}
#[args(kwargs = "**")]
fn with_padding(&mut self, kwargs: Option<&PyDict>) -> PyResult<()> {
fn enable_padding(&mut self, kwargs: Option<&PyDict>) -> PyResult<()> {
let mut direction = PaddingDirection::Right;
let mut pad_id: u32 = 0;
let mut pad_type_id: u32 = 0;
@ -148,7 +148,7 @@ impl Tokenizer {
Ok(())
}
fn without_padding(&mut self) {
fn no_padding(&mut self) {
self.tokenizer.with_padding(None);
}