mii/tokenizers
1
0
Fork 0
mirror of https://github.com/mii443/tokenizers.git synced 2025-08-22 16:25:30 +00:00
Code Issues Packages Projects Releases Wiki Activity

clean derive partial eq

Browse Source
This commit is contained in:
Arthur Zucker
2023-09-01 20:50:34 +00:00
parent fdef4a118b
commit 587748ab09
1 changed files with 1 additions and 12 deletions
Download Patch File Download Diff File Expand all files Collapse all files

13
tokenizers/src/tokenizer/added_vocabulary.rs
View File

@ -11,7 +11,7 @@ use std::collections::{HashMap, HashSet};
/// like: /// like:
/// - Whether they should only match single words /// - Whether they should only match single words
/// - Whether to include any whitespace on its left or right /// - Whether to include any whitespace on its left or right
#[derive(Debug, Clone, Serialize, Deserialize)] #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
pub struct AddedToken { pub struct AddedToken {
/// The content of the added token /// The content of the added token
pub content: String, pub content: String,
@ -91,17 +91,6 @@ impl std::hash::Hash for AddedToken {
self.content.hash(state); self.content.hash(state);
} }
} }
impl std::cmp::PartialEq for AddedToken {
fn eq(&self, other: &Self) -> bool {
self.content == other.content
&& self.special == other.special
&& self.lstrip == other.lstrip
&& self.rstrip == other.rstrip
&& self.normalized == other.normalized
&& self.single_word == other.single_word
}
}
impl std::cmp::Eq for AddedToken {}
type MatchingSet = (AhoCorasick, Vec<u32>); type MatchingSet = (AhoCorasick, Vec<u32>);