make content writable in python

bindings/python/src/tokenizer.rs

@@ -55,6 +55,8 @@ use crate::utils::{MaybeSizedIterator, PyBufferedIterator};
 ///         text. For example, with the added token ``"yesterday"``, and a normalizer in charge of
 ///         lowercasing the text, the token could be extract from the input ``"I saw a lion
 ///         Yesterday"``.
+///     special (:obj:`bool`, defaults to :obj:`False` with :meth:`~tokenizers.Tokenizer.add_tokens` and :obj:`False` with :meth:`~tokenizers.Tokenizer.add_special_tokens`):
+///         Defines whether this token should be skipped when decoding.
 ///
 #[pyclass(dict, module = "tokenizers", name = "AddedToken")]
 pub struct PyAddedToken {
@@ -179,6 +181,12 @@ impl PyAddedToken {
         &self.content
     }
 
+    /// Set the content of this :obj:`AddedToken`
+    #[setter]
+    fn set_content(&self, content: String){
+        self.get_token().content = content
+    }
+
     /// Get the value of the :obj:`rstrip` option
     #[getter]
     fn get_rstrip(&self) -> bool {

bindings/python/tests/bindings/test_tokenizer.py

@@ -16,6 +16,7 @@ from ..utils import bert_files, data_dir, multiprocessing_with_parallelism, robe
 class TestAddedToken:
     def test_instantiate_with_content_only(self):
         added_token = AddedToken("<mask>")
+        added_token.content = "<MASK>"
         assert type(added_token) == AddedToken
         assert str(added_token) == "<mask>"
         assert (