mii/tokenizers
1
0
Fork 0
mirror of https://github.com/mii443/tokenizers.git synced 2025-09-02 07:19:24 +00:00
Code Issues Packages Projects Releases Wiki Activity

Python - Update some naming

Browse Source
This commit is contained in:
Anthony MOI
2020-01-08 09:54:03 -05:00
parent 45c6382c35
commit fc56f8d186
2 changed files with 18 additions and 18 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
bindings/python/src/tokenizer.rs
View File

@ -56,7 +56,7 @@ impl Tokenizer {
} }
#[args(kwargs = "**")] #[args(kwargs = "**")]
fn with_truncation(&mut self, max_length: usize, kwargs: Option<&PyDict>) -> PyResult<()> { fn enable_truncation(&mut self, max_length: usize, kwargs: Option<&PyDict>) -> PyResult<()> {
let mut stride = 0; let mut stride = 0;
let mut strategy = TruncationStrategy::LongestFirst; let mut strategy = TruncationStrategy::LongestFirst;
@ -93,12 +93,12 @@ impl Tokenizer {
Ok(()) Ok(())
} }
fn without_truncation(&mut self) { fn no_truncation(&mut self) {
self.tokenizer.with_truncation(None); self.tokenizer.with_truncation(None);
} }
#[args(kwargs = "**")] #[args(kwargs = "**")]
fn with_padding(&mut self, kwargs: Option<&PyDict>) -> PyResult<()> { fn enable_padding(&mut self, kwargs: Option<&PyDict>) -> PyResult<()> {
let mut direction = PaddingDirection::Right; let mut direction = PaddingDirection::Right;
let mut pad_id: u32 = 0; let mut pad_id: u32 = 0;
let mut pad_type_id: u32 = 0; let mut pad_type_id: u32 = 0;
@ -148,7 +148,7 @@ impl Tokenizer {
Ok(()) Ok(())
} }
fn without_padding(&mut self) { fn no_padding(&mut self) {
self.tokenizer.with_padding(None); self.tokenizer.with_padding(None);
} }

12
bindings/python/tokenizers/__init__.pyi
View File

@ -198,11 +198,11 @@ class Tokenizer:
""" """
pass pass
def with_truncation(self, def enable_truncation(self,
max_length: int, max_length: int,
stride: Optional[int], stride: Optional[int],
strategy: Optional[str]): strategy: Optional[str]):
""" Change the truncation options """ Enable the truncation
Args: Args:
max_length: unsigned int: max_length: unsigned int:
@ -217,17 +217,17 @@ class Tokenizer:
""" """
pass pass
def without_truncation(self): def no_truncation(self):
""" Disable truncation """ """ Disable truncation """
pass pass
def with_padding(self, def enable_padding(self,
direction: Optional[str] = "right", direction: Optional[str] = "right",
pad_id: Optional[int] = 0, pad_id: Optional[int] = 0,
pad_type_id: Optional[int] = 0, pad_type_id: Optional[int] = 0,
pad_token: Optional[str] = "[PAD]", pad_token: Optional[str] = "[PAD]",
max_length: Optional[int] = None) max_length: Optional[int] = None)
""" Change the padding strategy """ Enable the padding
Args: Args:
direction: (`optional`) str: direction: (`optional`) str:
@ -248,7 +248,7 @@ class Tokenizer:
""" """
pass pass
def without_padding(self): def no_padding(self):
""" Disable padding """ """ Disable padding """
pass pass