mii/tokenizers
1
0
Fork 0
mirror of https://github.com/mii443/tokenizers.git synced 2025-09-02 15:29:21 +00:00
Code Issues Packages Projects Releases Wiki Activity

fmt

Browse Source
This commit is contained in:
Arthur Zucker
2023-09-04 19:31:05 +00:00
parent a59bb76aa1
commit 9aab096da8
2 changed files with 3 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
tokenizers/src/tokenizer/added_vocabulary.rs
View File

@ -675,10 +675,9 @@ mod tests {
assert_eq!(vocab.get_vocab()["another_two"], 4); // Token idx not changed
// Just checking that we can set the content of the string in rust
let mut token:AddedToken = AddedToken::from("Hey", false);
let mut token: AddedToken = AddedToken::from("Hey", false);
token.content = "hey".to_string();
assert_eq!(token.content, "hey"); // Token was already there
}
#[test]