mirror of
https://github.com/mii443/tokenizers.git
synced 2025-08-22 16:25:30 +00:00
BPE save compatibility tweak
This commit is contained in:
@ -444,6 +444,7 @@ impl Model for BPE {
|
||||
.map(|(pair, (rank, _))| (pair, rank))
|
||||
.collect();
|
||||
merges.sort_unstable_by_key(|k| *k.1);
|
||||
merges_file.write_all(b"# Trained by `huggingface/tokenizers`\n")?:
|
||||
merges_file.write_all(
|
||||
&merges
|
||||
.into_iter()
|
||||
|
Reference in New Issue
Block a user