mii/tokenizers
1
0
Fork 0
mirror of https://github.com/mii443/tokenizers.git synced 2025-08-23 00:35:35 +00:00
Code Issues Packages Projects Releases Wiki Activity

styling

Browse Source
This commit is contained in:
Arthur Zucker
2023-09-04 20:54:55 +00:00
parent 058e34b421
commit 93b37f36dc
1 changed files with 1 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
bindings/python/tests/bindings/test_tokenizer.py
View File

@ -20,7 +20,7 @@ class TestAddedToken:
assert added_token.content == "<MASK>"
assert type(added_token) == AddedToken
added_token.content = added_token.content.lower()
assert added_token.special == False
added_token.special = True
assert added_token.special == True