mii/tokenizers
1
0
Fork 0
mirror of https://github.com/mii443/tokenizers.git synced 2025-08-23 00:35:35 +00:00
Code Issues Packages Projects Releases Wiki Activity

Python - Improve AddedToken repr

Browse Source
This commit is contained in:
Anthony MOI
2020-03-30 14:38:34 -04:00
parent b055b77b54
commit 477037fd6b
2 changed files with 10 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
bindings/python/src/tokenizer.rs
View File

@ -74,9 +74,17 @@ impl PyObjectProtocol for AddedToken {
} }
fn __repr__(&self) -> PyResult<String> { fn __repr__(&self) -> PyResult<String> {
let bool_to_python = |p| match p {
true => "True",
false => "False",
};
Ok(format!( Ok(format!(
"AddedToken(\"{}\", rstrip={}, lstrip={}, single_word={})", "AddedToken(\"{}\", rstrip={}, lstrip={}, single_word={})",
self.token.content, self.token.rstrip, self.token.lstrip, self.token.single_word self.token.content,
bool_to_python(self.token.rstrip),
bool_to_python(self.token.lstrip),
bool_to_python(self.token.single_word)
)) ))
} }
} }

2
bindings/python/tests/bindings/test_tokenizer.py
View File

@ -9,7 +9,7 @@ class TestAddedToken:
assert str(added_token) == "<mask>" assert str(added_token) == "<mask>"
assert ( assert (
repr(added_token) repr(added_token)
== 'AddedToken("<mask>", rstrip=false, lstrip=false, single_word=false)' == 'AddedToken("<mask>", rstrip=False, lstrip=False, single_word=False)'
) )
assert added_token.rstrip == False assert added_token.rstrip == False
assert added_token.lstrip == False assert added_token.lstrip == False