BPE save compatibility tweak

This commit is contained in:
Julien Chaumond
2020-01-08 19:41:11 +00:00
parent bbe31f9237
commit cbe13e0aee

View File

@ -444,6 +444,7 @@ impl Model for BPE {
.map(|(pair, (rank, _))| (pair, rank))
.collect();
merges.sort_unstable_by_key(|k| *k.1);
merges_file.write_all(b"# Trained by `huggingface/tokenizers`\n")?:
merges_file.write_all(
&merges
.into_iter()