mii/tokenizers
1
0
Fork 0
mirror of https://github.com/mii443/tokenizers.git synced 2025-08-22 16:25:30 +00:00
Code Issues Packages Projects Releases Wiki Activity

Add fuse_unk option to SentencePieceBPETokenizer (#574)

Browse Source 
* Add fuse_unk option to SentencePieceBPETokenizer

* Fix style

Co-authored-by: Anthony MOI <m.anthony.moi@gmail.com>
This commit is contained in:
devfon
2021-01-13 06:07:59 +09:00
committed by GitHub
parent 91dae1de15
commit b9c6bea75e
1 changed files with 4 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
bindings/python/py_src/tokenizers/implementations/sentencepiece_bpe.py
View File

@ -20,9 +20,12 @@ class SentencePieceBPETokenizer(BaseTokenizer):
replacement: str = "▁",
add_prefix_space: bool = True,
dropout: Optional[float] = None,
fuse_unk: Optional[bool] = False,
):
if vocab is not None and merges is not None:
tokenizer = Tokenizer(BPE(vocab, merges, dropout=dropout, unk_token=unk_token))
tokenizer = Tokenizer(
BPE(vocab, merges, dropout=dropout, unk_token=unk_token, fuse_unk=fuse_unk)
)
else:
tokenizer = Tokenizer(BPE())