mirror of
https://github.com/mii443/tokenizers.git
synced 2025-08-22 16:25:30 +00:00
Going back for not
fuse_unk by default for BPE, but add a flag to
enable it.
This commit is contained in:
committed by
Anthony MOI
parent
940f8bd8fa
commit
9d3a93db5b
@ -106,6 +106,7 @@ class SpmConverter(Converter):
|
||||
out_vocab_filename,
|
||||
out_merge_filename,
|
||||
unk_token=proto.trainer_spec.unk_piece,
|
||||
fuse_unk=True,
|
||||
)
|
||||
)
|
||||
finally:
|
||||
|
Reference in New Issue
Block a user