mirror of
https://github.com/mii443/tokenizers.git
synced 2025-08-23 00:35:35 +00:00
make special editable as well
This commit is contained in:
@ -216,6 +216,12 @@ impl PyAddedToken {
|
||||
self.get_token().special
|
||||
}
|
||||
|
||||
/// Set the value of the :obj:`special` option
|
||||
#[setter]
|
||||
fn set_special(&mut self, special: bool) {
|
||||
self.special = special;
|
||||
}
|
||||
|
||||
fn __str__(&self) -> PyResult<&str> {
|
||||
Ok(&self.content)
|
||||
}
|
||||
|
@ -20,6 +20,10 @@ class TestAddedToken:
|
||||
assert added_token.content == "<MASK>"
|
||||
assert type(added_token) == AddedToken
|
||||
added_token.content = added_token.content.lower()
|
||||
|
||||
assert added_token.special == False
|
||||
added_token.special = True
|
||||
assert added_token.special == True
|
||||
|
||||
assert str(added_token) == "<mask>"
|
||||
assert (
|
||||
|
Reference in New Issue
Block a user