diff --git a/tokenizers/src/models/mod.rs b/tokenizers/src/models/mod.rs index 9dd3dc96..7c12ca4d 100644 --- a/tokenizers/src/models/mod.rs +++ b/tokenizers/src/models/mod.rs @@ -252,7 +252,11 @@ mod tests { r#"{"type":"BPE","dropout":null,"unk_token":"","continuing_subword_prefix":null,"end_of_word_suffix":null,"fuse_unk":false,"byte_fallback":false,"ignore_merges":true,"vocab":{"":0,"a":1,"b":2,"ab":3},"merges":["a b"]}"# ); let reconstructed = serde_json::from_str(&data).unwrap(); + assert_eq!(model, reconstructed); + // Legacy check, type is not necessary. + let legacy = r#"{"dropout":null,"unk_token":"","continuing_subword_prefix":null,"end_of_word_suffix":null,"fuse_unk":false,"byte_fallback":false,"ignore_merges":true,"vocab":{"":0,"a":1,"b":2,"ab":3},"merges":["a b"]}"#; + let reconstructed = serde_json::from_str(&legacy).unwrap(); assert_eq!(model, reconstructed); let invalid = r#"{"type":"BPE","dropout":null,"unk_token":"","continuing_subword_prefix":null,"end_of_word_suffix":null,"fuse_unk":false,"byte_fallback":false,"ignore_merges":true,"vocab":{"":0,"a":1,"b":2,"ab":3},"merges":["a b c"]}"#;