mii/tokenizers
1
0
Fork 0
mirror of https://github.com/mii443/tokenizers.git synced 2025-08-23 00:35:35 +00:00
Code Issues Packages Projects Releases Wiki Activity

revise type specification (#1289)

Browse Source
This commit is contained in:
Hiroshi Matsuda
2023-07-06 23:36:48 +09:00
committed by GitHub
parent 864135bef1
commit 26659de473
1 changed files with 1 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
bindings/python/py_src/tokenizers/implementations/sentencepiece_unigram.py
View File

@ -16,7 +16,7 @@ class SentencePieceUnigramTokenizer(BaseTokenizer):
def __init__( def __init__(
self, self,
vocab: Optional[str] = None, vocab: Optional[List[Tuple[str, float]]] = None,
replacement: str = "▁", replacement: str = "▁",
add_prefix_space: bool = True, add_prefix_space: bool = True,
): ):