mirror of
https://github.com/mii443/tokenizers.git
synced 2025-08-22 16:25:30 +00:00
Fix type notation of merges in BPE Python binding (#1766)
This commit is contained in:
@ -16,7 +16,7 @@ class ByteLevelBPETokenizer(BaseTokenizer):
|
|||||||
def __init__(
|
def __init__(
|
||||||
self,
|
self,
|
||||||
vocab: Optional[Union[str, Dict[str, int]]] = None,
|
vocab: Optional[Union[str, Dict[str, int]]] = None,
|
||||||
merges: Optional[Union[str, Dict[Tuple[int, int], Tuple[int, int]]]] = None,
|
merges: Optional[Union[str, List[Tuple[str, str]]]] = None,
|
||||||
add_prefix_space: bool = False,
|
add_prefix_space: bool = False,
|
||||||
lowercase: bool = False,
|
lowercase: bool = False,
|
||||||
dropout: Optional[float] = None,
|
dropout: Optional[float] = None,
|
||||||
|
@ -25,7 +25,7 @@ class CharBPETokenizer(BaseTokenizer):
|
|||||||
def __init__(
|
def __init__(
|
||||||
self,
|
self,
|
||||||
vocab: Optional[Union[str, Dict[str, int]]] = None,
|
vocab: Optional[Union[str, Dict[str, int]]] = None,
|
||||||
merges: Optional[Union[str, Dict[Tuple[int, int], Tuple[int, int]]]] = None,
|
merges: Optional[Union[str, List[Tuple[str, str]]]] = None,
|
||||||
unk_token: Union[str, AddedToken] = "<unk>",
|
unk_token: Union[str, AddedToken] = "<unk>",
|
||||||
suffix: str = "</w>",
|
suffix: str = "</w>",
|
||||||
dropout: Optional[float] = None,
|
dropout: Optional[float] = None,
|
||||||
|
@ -16,7 +16,7 @@ class SentencePieceBPETokenizer(BaseTokenizer):
|
|||||||
def __init__(
|
def __init__(
|
||||||
self,
|
self,
|
||||||
vocab: Optional[Union[str, Dict[str, int]]] = None,
|
vocab: Optional[Union[str, Dict[str, int]]] = None,
|
||||||
merges: Optional[Union[str, Dict[Tuple[int, int], Tuple[int, int]]]] = None,
|
merges: Optional[Union[str, List[Tuple[str, str]]]] = None,
|
||||||
unk_token: Union[str, AddedToken] = "<unk>",
|
unk_token: Union[str, AddedToken] = "<unk>",
|
||||||
replacement: str = "▁",
|
replacement: str = "▁",
|
||||||
add_prefix_space: bool = True,
|
add_prefix_space: bool = True,
|
||||||
|
Reference in New Issue
Block a user