mii/tokenizers
1
0
Fork 0
mirror of https://github.com/mii443/tokenizers.git synced 2025-08-22 16:25:30 +00:00
Code Issues Packages Projects Releases Wiki Activity

Clippy.

Browse Source
This commit is contained in:
Nicolas Patry
2024-08-06 10:08:35 +02:00
parent 5fb8a2320c
commit 99a48dcb46
1 changed files with 2 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
tokenizers/src/models/mod.rs
View File

@ -256,12 +256,12 @@ mod tests {
// Legacy check, type is not necessary. // Legacy check, type is not necessary.
let legacy = r#"{"dropout":null,"unk_token":"<unk>","continuing_subword_prefix":null,"end_of_word_suffix":null,"fuse_unk":false,"byte_fallback":false,"ignore_merges":true,"vocab":{"<unk>":0,"a":1,"b":2,"ab":3},"merges":["a b"]}"#; let legacy = r#"{"dropout":null,"unk_token":"<unk>","continuing_subword_prefix":null,"end_of_word_suffix":null,"fuse_unk":false,"byte_fallback":false,"ignore_merges":true,"vocab":{"<unk>":0,"a":1,"b":2,"ab":3},"merges":["a b"]}"#;
let reconstructed = serde_json::from_str(&legacy).unwrap(); let reconstructed = serde_json::from_str(legacy).unwrap();
assert_eq!(model, reconstructed); assert_eq!(model, reconstructed);
let invalid = r#"{"type":"BPE","dropout":null,"unk_token":"<unk>","continuing_subword_prefix":null,"end_of_word_suffix":null,"fuse_unk":false,"byte_fallback":false,"ignore_merges":true,"vocab":{"<unk>":0,"a":1,"b":2,"ab":3},"merges":["a b c"]}"#; let invalid = r#"{"type":"BPE","dropout":null,"unk_token":"<unk>","continuing_subword_prefix":null,"end_of_word_suffix":null,"fuse_unk":false,"byte_fallback":false,"ignore_merges":true,"vocab":{"<unk>":0,"a":1,"b":2,"ab":3},"merges":["a b c"]}"#;
let reconstructed: std::result::Result<ModelWrapper, serde_json::Error> = let reconstructed: std::result::Result<ModelWrapper, serde_json::Error> =
serde_json::from_str(&invalid); serde_json::from_str(invalid);
match reconstructed { match reconstructed {
Err(err) => assert_eq!( Err(err) => assert_eq!(
err.to_string(), err.to_string(),