mii/tokenizers
1
0
Fork 0
mirror of https://github.com/mii443/tokenizers.git synced 2025-09-02 15:29:21 +00:00
Code Issues Packages Projects Releases Wiki Activity

Python - Update some naming

Browse Source
This commit is contained in:
Anthony MOI
2020-01-08 09:54:03 -05:00
parent 45c6382c35
commit fc56f8d186
2 changed files with 18 additions and 18 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
bindings/python/src/tokenizer.rs
View File

@ -56,7 +56,7 @@ impl Tokenizer {
}
#[args(kwargs = "**")]
fn with_truncation(&mut self, max_length: usize, kwargs: Option<&PyDict>) -> PyResult<()> {
fn enable_truncation(&mut self, max_length: usize, kwargs: Option<&PyDict>) -> PyResult<()> {
let mut stride = 0;
let mut strategy = TruncationStrategy::LongestFirst;
@ -93,12 +93,12 @@ impl Tokenizer {
Ok(())
}
fn without_truncation(&mut self) {
fn no_truncation(&mut self) {
self.tokenizer.with_truncation(None);
}
#[args(kwargs = "**")]
fn with_padding(&mut self, kwargs: Option<&PyDict>) -> PyResult<()> {
fn enable_padding(&mut self, kwargs: Option<&PyDict>) -> PyResult<()> {
let mut direction = PaddingDirection::Right;
let mut pad_id: u32 = 0;
let mut pad_type_id: u32 = 0;
@ -148,7 +148,7 @@ impl Tokenizer {
Ok(())
}
fn without_padding(&mut self) {
fn no_padding(&mut self) {
self.tokenizer.with_padding(None);
}

28
bindings/python/tokenizers/__init__.pyi
View File

@ -198,11 +198,11 @@ class Tokenizer:
"""
pass
def with_truncation(self,
max_length: int,
stride: Optional[int],
strategy: Optional[str]):
""" Change the truncation options
def enable_truncation(self,
max_length: int,
stride: Optional[int],
strategy: Optional[str]):
""" Enable the truncation
Args:
max_length: unsigned int:
@ -217,17 +217,17 @@ class Tokenizer:
"""
pass
def without_truncation(self):
def no_truncation(self):
""" Disable truncation """
pass
def with_padding(self,
direction: Optional[str] = "right",
pad_id: Optional[int] = 0,
pad_type_id: Optional[int] = 0,
pad_token: Optional[str] = "[PAD]",
max_length: Optional[int] = None)
""" Change the padding strategy
def enable_padding(self,
direction: Optional[str] = "right",
pad_id: Optional[int] = 0,
pad_type_id: Optional[int] = 0,
pad_token: Optional[str] = "[PAD]",
max_length: Optional[int] = None)
""" Enable the padding
Args:
direction: (`optional`) str:
@ -248,7 +248,7 @@ class Tokenizer:
"""
pass
def without_padding(self):
def no_padding(self):
""" Disable padding """
pass