mii/tokenizers
1
0
Fork 0
mirror of https://github.com/mii443/tokenizers.git synced 2025-08-22 16:25:30 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix some tests

Browse Source
This commit is contained in:
Arthur Zucker
2023-09-01 18:48:50 +00:00
parent 6cca5716af
commit 2dca476810
1 changed files with 3 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
tokenizers/src/tokenizer/added_vocabulary.rs
View File

@ -607,7 +607,7 @@ mod tests {
assert_eq!(vocab.add_tokens(&[added_token.clone()], &model, normalizer),1);
assert_eq!(vocab.len(), 3);
assert_eq!(vocab.get_vocab_r()[&1], added_token);
assert_eq!(vocab.get_vocab_r()[&0], added_token);
}
#[test]
@ -652,8 +652,8 @@ mod tests {
(3,AddedToken::from("added_token_2", true)),
(0,AddedToken::from("test", true)),
]));
assert!(!vocab.added_tokens_map.contains_key("test"));
assert!(vocab.added_tokens_map.contains_key("test"));
assert!(vocab.added_tokens_map_r.contains_key(&0));
}
#[test]