mii/tokenizers
1
0
Fork 0
mirror of https://github.com/mii443/tokenizers.git synced 2025-08-22 16:25:30 +00:00
Code Issues Packages Projects Releases Wiki Activity

fmt

Browse Source
This commit is contained in:
Arthur Zucker
2023-09-01 20:48:47 +00:00
parent d1566a9ecc
commit fdef4a118b
1 changed files with 10 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
tokenizers/src/tokenizer/added_vocabulary.rs
View File

@ -93,7 +93,12 @@ impl std::hash::Hash for AddedToken {
} }
impl std::cmp::PartialEq for AddedToken { impl std::cmp::PartialEq for AddedToken {
fn eq(&self, other: &Self) -> bool { fn eq(&self, other: &Self) -> bool {
self.content == other.content && self.special == other.special && self.lstrip == other.lstrip && self.rstrip == other.rstrip && self.normalized == other.normalized && self.single_word == other.single_word self.content == other.content
&& self.special == other.special
&& self.lstrip == other.lstrip
&& self.rstrip == other.rstrip
&& self.normalized == other.normalized
&& self.single_word == other.single_word
} }
} }
impl std::cmp::Eq for AddedToken {} impl std::cmp::Eq for AddedToken {}
@ -673,7 +678,10 @@ mod tests {
assert_eq!(vocab.get_vocab()["another_two"], 4); // New token was added, but the index is not the length of the vocab assert_eq!(vocab.get_vocab()["another_two"], 4); // New token was added, but the index is not the length of the vocab
// Let's add an already added token again // Let's add an already added token again
assert_eq!(vocab.add_special_tokens(&[AddedToken::from("another_two", true)], &model, normalizer), 1); assert_eq!(
vocab.add_special_tokens(&[AddedToken::from("another_two", true)], &model, normalizer),
1
);
assert_eq!(vocab.len(), 5); // Token was already there assert_eq!(vocab.len(), 5); // Token was already there
assert_eq!(vocab.get_vocab()["another_two"], 4); // Token idx not changed assert_eq!(vocab.get_vocab()["another_two"], 4); // Token idx not changed
} }