mii/tokenizers
1
0
Fork 0
mirror of https://github.com/mii443/tokenizers.git synced 2025-08-23 16:49:27 +00:00
Code Issues Packages Projects Releases Wiki Activity

make content writable in python

Browse Source
This commit is contained in:
Arthur Zucker
2023-09-04 18:18:21 +00:00
parent d9829cdc6e
commit a53dff9bc5
2 changed files with 9 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
bindings/python/src/tokenizer.rs
View File

@ -55,6 +55,8 @@ use crate::utils::{MaybeSizedIterator, PyBufferedIterator};
/// text. For example, with the added token ``"yesterday"``, and a normalizer in charge of /// text. For example, with the added token ``"yesterday"``, and a normalizer in charge of
/// lowercasing the text, the token could be extract from the input ``"I saw a lion /// lowercasing the text, the token could be extract from the input ``"I saw a lion
/// Yesterday"``. /// Yesterday"``.
/// special (:obj:`bool`, defaults to :obj:`False` with :meth:`~tokenizers.Tokenizer.add_tokens` and :obj:`False` with :meth:`~tokenizers.Tokenizer.add_special_tokens`):
/// Defines whether this token should be skipped when decoding.
/// ///
#[pyclass(dict, module = "tokenizers", name = "AddedToken")] #[pyclass(dict, module = "tokenizers", name = "AddedToken")]
pub struct PyAddedToken { pub struct PyAddedToken {
@ -179,6 +181,12 @@ impl PyAddedToken {
&self.content &self.content
} }
/// Set the content of this :obj:`AddedToken`
#[setter]
fn set_content(&self, content: String){
self.get_token().content = content
}
/// Get the value of the :obj:`rstrip` option /// Get the value of the :obj:`rstrip` option
#[getter] #[getter]
fn get_rstrip(&self) -> bool { fn get_rstrip(&self) -> bool {

1
bindings/python/tests/bindings/test_tokenizer.py
View File

@ -16,6 +16,7 @@ from ..utils import bert_files, data_dir, multiprocessing_with_parallelism, robe
class TestAddedToken: class TestAddedToken:
def test_instantiate_with_content_only(self): def test_instantiate_with_content_only(self):
added_token = AddedToken("<mask>") added_token = AddedToken("<mask>")
added_token.content = "<MASK>"
assert type(added_token) == AddedToken assert type(added_token) == AddedToken
assert str(added_token) == "<mask>" assert str(added_token) == "<mask>"
assert ( assert (