mii/tokenizers
1
0
Fork 0
mirror of https://github.com/mii443/tokenizers.git synced 2025-08-31 04:29:21 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix add_special_tokens

Browse Source
This commit is contained in:
thomwolf
2020-01-08 11:48:37 +01:00
parent b16ee75b97
commit af6a685664
1 changed files with 1 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
bindings/python/tokenizers/implementations/base_tokenizer.py
View File

@ -68,7 +68,6 @@ class BaseTokenizer:
""" Disable truncation """ """ Disable truncation """
return self._tokenizer.without_truncation() return self._tokenizer.without_truncation()
def add_tokens(self, tokens: List[Union[str, Tuple[str, bool]]]) -> int: def add_tokens(self, tokens: List[Union[str, Tuple[str, bool]]]) -> int:
""" Add the given tokens to the vocabulary """ Add the given tokens to the vocabulary
@ -97,7 +96,7 @@ class BaseTokenizer:
Returns: Returns:
The number of tokens that were added to the vocabulary The number of tokens that were added to the vocabulary
""" """
return self._tokenizer.add_special_tokens(tokens) return self._tokenizer.add_special_tokens(special_tokens)
def encode(self, sequence: str, pair: Optional[str] = None) -> Encoding: def encode(self, sequence: str, pair: Optional[str] = None) -> Encoding:
""" Encode the given sequence """ Encode the given sequence