mii/tokenizers
1
0
Fork 0
mirror of https://github.com/mii443/tokenizers.git synced 2025-08-22 16:25:30 +00:00
Code Issues Packages Projects Releases Wiki Activity

Fix type notation of merges in BPE Python binding (#1766)

Browse Source
This commit is contained in:
Kokū
2025-05-27 15:23:58 +09:00
committed by GitHub
parent f1faec1756
commit cc01186fd7
3 changed files with 3 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
bindings/python/py_src/tokenizers/implementations/byte_level_bpe.py
View File

@ -16,7 +16,7 @@ class ByteLevelBPETokenizer(BaseTokenizer):
def __init__( def __init__(
self, self,
vocab: Optional[Union[str, Dict[str, int]]] = None, vocab: Optional[Union[str, Dict[str, int]]] = None,
merges: Optional[Union[str, Dict[Tuple[int, int], Tuple[int, int]]]] = None, merges: Optional[Union[str, List[Tuple[str, str]]]] = None,
add_prefix_space: bool = False, add_prefix_space: bool = False,
lowercase: bool = False, lowercase: bool = False,
dropout: Optional[float] = None, dropout: Optional[float] = None,

2
bindings/python/py_src/tokenizers/implementations/char_level_bpe.py
View File

@ -25,7 +25,7 @@ class CharBPETokenizer(BaseTokenizer):
def __init__( def __init__(
self, self,
vocab: Optional[Union[str, Dict[str, int]]] = None, vocab: Optional[Union[str, Dict[str, int]]] = None,
merges: Optional[Union[str, Dict[Tuple[int, int], Tuple[int, int]]]] = None, merges: Optional[Union[str, List[Tuple[str, str]]]] = None,
unk_token: Union[str, AddedToken] = "<unk>", unk_token: Union[str, AddedToken] = "<unk>",
suffix: str = "</w>", suffix: str = "</w>",
dropout: Optional[float] = None, dropout: Optional[float] = None,

2
bindings/python/py_src/tokenizers/implementations/sentencepiece_bpe.py
View File

@ -16,7 +16,7 @@ class SentencePieceBPETokenizer(BaseTokenizer):
def __init__( def __init__(
self, self,
vocab: Optional[Union[str, Dict[str, int]]] = None, vocab: Optional[Union[str, Dict[str, int]]] = None,
merges: Optional[Union[str, Dict[Tuple[int, int], Tuple[int, int]]]] = None, merges: Optional[Union[str, List[Tuple[str, str]]]] = None,
unk_token: Union[str, AddedToken] = "<unk>", unk_token: Union[str, AddedToken] = "<unk>",
replacement: str = "▁", replacement: str = "▁",
add_prefix_space: bool = True, add_prefix_space: bool = True,