mii/tokenizers
1
0
Fork 0
mirror of https://github.com/mii443/tokenizers.git synced 2025-12-12 05:18:39 +00:00
Code Issues Packages Projects Releases Wiki Activity

Throw a more meaningful error when provided python input is None.

Browse Source
This commit is contained in:
Morgan Funtowicz
2020-03-12 10:59:05 +01:00
parent 5673b00724
commit 5ed1f26c71
1 changed files with 18 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

18
bindings/python/tokenizers/implementations/base_tokenizer.py
View File

@@ -143,6 +143,10 @@ class BaseTokenizer:
Returns: Returns:
An Encoding An Encoding
""" """
if sequence is None:
raise ValueError(
"Input is not valid. Should be a string, a list/tuple of strings or a list/tuple of integers."
)
return self._tokenizer.encode(sequence, pair, add_special_tokens) return self._tokenizer.encode(sequence, pair, add_special_tokens)
def encode_batch( def encode_batch(
@@ -161,6 +165,12 @@ class BaseTokenizer:
Returns: Returns:
A list of Encoding A list of Encoding
""" """
if sequences is None:
raise ValueError(
"Input is not valid. Should be a string, a list/tuple of strings or a list/tuple of integers."
)
return self._tokenizer.encode_batch(sequences, add_special_tokens) return self._tokenizer.encode_batch(sequences, add_special_tokens)
def decode(self, ids: List[int], skip_special_tokens: Optional[bool] = True) -> str: def decode(self, ids: List[int], skip_special_tokens: Optional[bool] = True) -> str:
@@ -176,6 +186,10 @@ class BaseTokenizer:
Returns: Returns:
The decoded string The decoded string
""" """
if ids is None:
raise ValueError(
"Input is not valid. Should be a string, a list/tuple of strings or a list/tuple of integers."
)
return self._tokenizer.decode(ids, skip_special_tokens=skip_special_tokens) return self._tokenizer.decode(ids, skip_special_tokens=skip_special_tokens)
def decode_batch( def decode_batch(
@@ -193,6 +207,10 @@ class BaseTokenizer:
Returns: Returns:
A list of decoded strings A list of decoded strings
""" """
if sequences is None:
raise ValueError(
"Input is not valid. Should be a string, a list/tuple of strings or a list/tuple of integers."
)
return self._tokenizer.decode_batch(sequences, skip_special_tokens=skip_special_tokens) return self._tokenizer.decode_batch(sequences, skip_special_tokens=skip_special_tokens)
def token_to_id(self, token: str) -> Optional[int]: def token_to_id(self, token: str) -> Optional[int]: