updates

bindings/python/src/tokenizer.rs

@@ -183,8 +183,8 @@ impl PyAddedToken {
 
     /// Set the content of this :obj:`AddedToken`
     #[setter]
-    fn set_content(&self, content: String){
-        self.get_token().content = content
+    fn set_content(&mut self, content: String) {
+        self.content = content.into();
     }
 
     /// Get the value of the :obj:`rstrip` option

bindings/python/tests/bindings/test_tokenizer.py

@@ -17,7 +17,10 @@ class TestAddedToken:
     def test_instantiate_with_content_only(self):
         added_token = AddedToken("<mask>")
         added_token.content = "<MASK>"
+        assert added_token.content == "<MASK>"
         assert type(added_token) == AddedToken
+        added_token.content = added_token.content.lower()
+        
         assert str(added_token) == "<mask>"
         assert (
             repr(added_token) == 'AddedToken("<mask>", rstrip=False, lstrip=False, single_word=False, normalized=True, special=False)'

tokenizers/src/tokenizer/added_vocabulary.rs

@@ -673,6 +673,12 @@ mod tests {
         );
         assert_eq!(vocab.len(), 5); // Token was already there
         assert_eq!(vocab.get_vocab()["another_two"], 4); // Token idx not changed
+
+        // Just checking that we can set the content of the string in rust
+        let mut token:AddedToken = AddedToken::from("Hey", false);
+        token.content = "hey".to_string();
+        assert_eq!(token.content, "hey"); // Token was already there
+
     }
 
     #[test]