From 058e34b4216c22a867f739e131e8280f9968ff12 Mon Sep 17 00:00:00 2001 From: Arthur Zucker Date: Mon, 4 Sep 2023 20:54:29 +0000 Subject: [PATCH] make special editable as well --- bindings/python/src/tokenizer.rs | 6 ++++++ bindings/python/tests/bindings/test_tokenizer.py | 4 ++++ tokenizers/src/tokenizer/added_vocabulary.rs | 4 ++++ 3 files changed, 14 insertions(+) diff --git a/bindings/python/src/tokenizer.rs b/bindings/python/src/tokenizer.rs index 19e340d0..a8ae60c7 100644 --- a/bindings/python/src/tokenizer.rs +++ b/bindings/python/src/tokenizer.rs @@ -216,6 +216,12 @@ impl PyAddedToken { self.get_token().special } + /// Set the value of the :obj:`special` option + #[setter] + fn set_special(&mut self, special: bool) { + self.special = special; + } + fn __str__(&self) -> PyResult<&str> { Ok(&self.content) } diff --git a/bindings/python/tests/bindings/test_tokenizer.py b/bindings/python/tests/bindings/test_tokenizer.py index 3b0d9cab..89c097b0 100644 --- a/bindings/python/tests/bindings/test_tokenizer.py +++ b/bindings/python/tests/bindings/test_tokenizer.py @@ -20,6 +20,10 @@ class TestAddedToken: assert added_token.content == "" assert type(added_token) == AddedToken added_token.content = added_token.content.lower() + + assert added_token.special == False + added_token.special = True + assert added_token.special == True assert str(added_token) == "" assert ( diff --git a/tokenizers/src/tokenizer/added_vocabulary.rs b/tokenizers/src/tokenizer/added_vocabulary.rs index 38b623aa..58b21f5e 100644 --- a/tokenizers/src/tokenizer/added_vocabulary.rs +++ b/tokenizers/src/tokenizer/added_vocabulary.rs @@ -678,6 +678,10 @@ mod tests { let mut token: AddedToken = AddedToken::from("Hey", false); token.content = "hey".to_string(); assert_eq!(token.content, "hey"); // Token was already there + + token.special = true; + assert_eq!(token.special, true); // Token was already there + } #[test]