mii/tokenizers
1
0
Fork 0
mirror of https://github.com/mii443/tokenizers.git synced 2025-08-22 16:25:30 +00:00
Code Issues Packages Projects Releases Wiki Activity

Update model.rs (#1166)

Browse Source
This commit is contained in:
Thomas Wang
2023-02-28 17:35:57 +01:00
committed by GitHub
parent fa66caf0ab
commit ac552ff8b9
1 changed files with 1 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
tokenizers/src/models/bpe/model.rs
View File

@ -491,7 +491,7 @@ impl Model for BPE {
.map(|(pair, (rank, _))| (pair, rank))
.collect();
merges.sort_unstable_by_key(|k| *k.1);
merges_file.write_all(b"#version: 0.2 - Trained by `huggingface/tokenizers`\n")?;
merges_file.write_all(b"#version: 0.2\n")?;
merges_file.write_all(
&merges
.into_iter()