make special editable as well

2025-08-23 00:35:35 +00:00 · 2023-09-04 20:54:29 +00:00
parent 2291c89896
commit 058e34b421
3 changed files with 14 additions and 0 deletions
--- a/bindings/python/tests/bindings/test_tokenizer.py
+++ b/bindings/python/tests/bindings/test_tokenizer.py
@ -20,6 +20,10 @@ class TestAddedToken:
        assert added_token.content == "<MASK>"
        assert type(added_token) == AddedToken
        added_token.content = added_token.content.lower()
+        
+        assert added_token.special == False
+        added_token.special = True
+        assert added_token.special == True

        assert str(added_token) == "<mask>"
        assert (