mii/tokenizers
1
0
Fork 0
mirror of https://github.com/mii443/tokenizers.git synced 2025-12-03 11:18:29 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix more tests

Browse Source
This commit is contained in:
Arthur Zucker
2023-09-04 17:22:27 +00:00
parent 39bd27e673
commit d9829cdc6e
2 changed files with 2 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
bindings/python/src/trainers.rs
View File

@@ -226,7 +226,7 @@ impl PyBpeTrainer {
if let Ok(content) = token.extract::<String>() {
Ok(tk::tokenizer::AddedToken::from(content, true))
} else if let Ok(mut token) = token.extract::<PyRefMut<PyAddedToken>>() {
token.special = true;
token.special = false;
Ok(token.get_token())
} else {
Err(exceptions::PyTypeError::new_err(

2
bindings/python/tests/bindings/test_tokenizer.py
View File

@@ -19,7 +19,7 @@ class TestAddedToken:
assert type(added_token) == AddedToken
assert str(added_token) == "<mask>"
assert (
repr(added_token) == 'AddedToken("<mask>", rstrip=False, lstrip=False, single_word=False, normalized=True)'
repr(added_token) == 'AddedToken("<mask>", rstrip=False, lstrip=False, single_word=False, normalized=True, special=False)'
)
assert added_token.rstrip == False
assert added_token.lstrip == False