Going back for not fuse_unk by default for BPE, but add a flag to

enable it.
2025-08-23 00:35:35 +00:00 · 2020-09-22 11:00:08 +02:00
parent 940f8bd8fa
commit 9d3a93db5b
6 changed files with 95 additions and 7 deletions
--- a/bindings/python/scripts/convert.py
+++ b/bindings/python/scripts/convert.py
@ -106,6 +106,7 @@ class SpmConverter(Converter):
                            out_vocab_filename,
                            out_merge_filename,
                            unk_token=proto.trainer_spec.unk_piece,
+                            fuse_unk=True,
                        )
                    )
                finally: