From 587748ab097f0f1c3b8f58be23fc72974255553c Mon Sep 17 00:00:00 2001
From: Arthur Zucker <arthur.zucker@gmail.com>
Date: Fri, 1 Sep 2023 20:50:34 +0000
Subject: [PATCH] clean derive partial eq

---
 tokenizers/src/tokenizer/added_vocabulary.rs | 13 +------------
 1 file changed, 1 insertion(+), 12 deletions(-)

diff --git a/tokenizers/src/tokenizer/added_vocabulary.rs b/tokenizers/src/tokenizer/added_vocabulary.rs
index 5ed77c6d..cd656cdf 100644
--- a/tokenizers/src/tokenizer/added_vocabulary.rs
+++ b/tokenizers/src/tokenizer/added_vocabulary.rs
@@ -11,7 +11,7 @@ use std::collections::{HashMap, HashSet};
 /// like:
 ///   - Whether they should only match single words
 ///   - Whether to include any whitespace on its left or right
-#[derive(Debug, Clone, Serialize, Deserialize)]
+#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
 pub struct AddedToken {
     /// The content of the added token
     pub content: String,
@@ -91,17 +91,6 @@ impl std::hash::Hash for AddedToken {
         self.content.hash(state);
     }
 }
-impl std::cmp::PartialEq for AddedToken {
-    fn eq(&self, other: &Self) -> bool {
-        self.content == other.content
-            && self.special == other.special
-            && self.lstrip == other.lstrip
-            && self.rstrip == other.rstrip
-            && self.normalized == other.normalized
-            && self.single_word == other.single_word
-    }
-}
-impl std::cmp::Eq for AddedToken {}
 
 type MatchingSet = (AhoCorasick, Vec<u32>);