mii/tokenizers
1
0
Fork 0
mirror of https://github.com/mii443/tokenizers.git synced 2025-08-23 00:35:35 +00:00
Code Issues Packages Projects Releases Wiki Activity

updates

Browse Source
This commit is contained in:
Arthur Zucker
2023-09-04 19:10:22 +00:00
parent a53dff9bc5
commit b117ac7f16
3 changed files with 11 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
bindings/python/src/tokenizer.rs
View File

@ -183,8 +183,8 @@ impl PyAddedToken {
/// Set the content of this :obj:`AddedToken`
#[setter]
fn set_content(&self, content: String){
self.get_token().content = content
fn set_content(&mut self, content: String) {
self.content = content.into();
}
/// Get the value of the :obj:`rstrip` option