mii/tokenizers
1
0
Fork 0
mirror of https://github.com/mii443/tokenizers.git synced 2025-08-22 16:25:30 +00:00
Code Issues Packages Projects Releases Wiki Activity

Fix some more warnings

Browse Source
This commit is contained in:
Anthony MOI
2020-02-01 11:06:14 -05:00
parent 661cc185a4
commit e9ecd5aeec
1 changed files with 5 additions and 11 deletions
Download Patch File Download Diff File Expand all files Collapse all files

16
tokenizers/src/models/bpe/model.rs
View File

@ -578,14 +578,12 @@ mod tests {
// Set up vocab file.
let mut vocab_file = NamedTempFile::new().unwrap();
vocab_file
.write_all("{\"a\": 0, \"b\": 1, \"c\": 2, \"ab\": 3}".as_bytes())
.write_all(b"{\"a\": 0, \"b\": 1, \"c\": 2, \"ab\": 3}")
.unwrap();
// Set up merges file.
let mut merges_file = NamedTempFile::new().unwrap();
merges_file
.write_all("#version: 0.2\na b".as_bytes())
.unwrap();
merges_file.write_all(b"#version: 0.2\na b").unwrap();
// Make sure we can instantiate a BPE model from the files.
let result = BPE::from_files(
@ -612,14 +610,12 @@ mod tests {
// Set up vocab file.
let mut vocab_file = NamedTempFile::new().unwrap();
vocab_file
.write_all("{\"a\": 0, \"b\": 1, \"c\": 2, \"ab\": 3}".as_bytes())
.write_all(b"{\"a\": 0, \"b\": 1, \"c\": 2, \"ab\": 3}")
.unwrap();
// Set up merges file.
let mut merges_file = NamedTempFile::new().unwrap();
merges_file
.write_all("#version: 0.2\na b\na d".as_bytes())
.unwrap();
merges_file.write_all(b"#version: 0.2\na b\na d").unwrap();
// Ensure the result of BPE::from_files is a MergeTokenOutOfVocabulary error.
match BPE::from_files(
@ -648,9 +644,7 @@ mod tests {
// Set up merges file with a bad line.
let mut merges_file = NamedTempFile::new().unwrap();
merges_file
.write_all("#version: 0.2\na b\nc".as_bytes())
.unwrap();
merges_file.write_all(b"#version: 0.2\na b\nc").unwrap();
// Ensure the result of BPE::from_files is a BadMerges error.
match BPE::from_files(