mii/tokenizers
1
0
Fork 0
mirror of https://github.com/mii443/tokenizers.git synced 2025-08-22 16:25:30 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix one test?

Browse Source
This commit is contained in:
Arthur Zucker
2023-09-01 18:42:30 +00:00
parent 345b4eba96
commit 6cca5716af
1 changed files with 1 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
tokenizers/src/tokenizer/added_vocabulary.rs
View File

@ -650,7 +650,7 @@ mod tests {
assert_eq!(*vocab.get_vocab_r(), HashMap::from([
(2,AddedToken::from("added_token_1", true)),
(3,AddedToken::from("added_token_2", true)),
(4,AddedToken::from("test", true)),
(0,AddedToken::from("test", true)),
]));
assert!(!vocab.added_tokens_map.contains_key("test"));