mii/tokenizers
1
0
Fork 0
mirror of https://github.com/mii443/tokenizers.git synced 2025-08-22 16:25:30 +00:00
Code Issues Packages Projects Releases Wiki Activity

Fixup + better compat

Browse Source
This commit is contained in:
Julien Chaumond
2020-01-08 20:52:28 +00:00
parent cbe13e0aee
commit 6697d65544
1 changed files with 2 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
tokenizers/src/models/bpe/model.rs
View File

@ -444,7 +444,7 @@ impl Model for BPE {
.map(|(pair, (rank, _))| (pair, rank))
.collect();
merges.sort_unstable_by_key(|k| *k.1);
merges_file.write_all(b"# Trained by `huggingface/tokenizers`\n")?:
merges_file.write_all(b"#version: 0.2 - Trained by `huggingface/tokenizers`\n")?;
merges_file.write_all(
&merges
.into_iter()
@ -551,7 +551,7 @@ mod tests {
.write_all("#version: 0.2\na b".as_bytes())
.unwrap();
// Make sure we can instatiate a BPE model from the files.
// Make sure we can instantiate a BPE model from the files.
let result = BPE::from_files(
vocab_file.path().to_str().unwrap(),
merges_file.path().to_str().unwrap(),