mii/tokenizers
1
0
Fork 0
mirror of https://github.com/mii443/tokenizers.git synced 2025-12-05 04:08:22 +00:00
Code Issues Packages Projects Releases Wiki Activity

Formatting after rebase.

Browse Source
This commit is contained in:
Nicolas Patry
2020-08-24 13:50:11 +02:00
parent 439305eea0
commit e974cfb1c9
4 changed files with 6 additions and 18 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
bindings/python/py_src/tokenizers/__init__.py
View File

@@ -8,8 +8,7 @@ TextInputSequence = str
PreTokenizedInputSequence = Union[List[str], Tuple[str]]
TextEncodeInput = Union[TextInputSequence, Tuple[TextInputSequence, TextInputSequence]]
PreTokenizedEncodeInput = Union[
PreTokenizedInputSequence,
Tuple[PreTokenizedInputSequence, PreTokenizedInputSequence],
PreTokenizedInputSequence, Tuple[PreTokenizedInputSequence, PreTokenizedInputSequence],
]
InputSequence = Union[TextInputSequence, PreTokenizedInputSequence]

5
bindings/python/py_src/tokenizers/implementations/sentencepiece_unigram.py
View File

@@ -13,10 +13,7 @@ class SentencePieceUnigramTokenizer(BaseTokenizer):
"""
def __init__(
self,
vocab: Optional[str] = None,
replacement: str = "▁",
add_prefix_space: bool = True,
self, vocab: Optional[str] = None, replacement: str = "▁", add_prefix_space: bool = True,
):
if vocab is not None:
tokenizer = Tokenizer(Unigram(vocab))