mii/tokenizers
1
0
Fork 0
mirror of https://github.com/mii443/tokenizers.git synced 2025-12-18 06:19:14 +00:00
Code Issues Packages Projects Releases Wiki Activity

updates

Browse Source
This commit is contained in:
Arthur Zucker
2023-09-04 19:10:22 +00:00
parent a53dff9bc5
commit b117ac7f16
3 changed files with 11 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
bindings/python/tests/bindings/test_tokenizer.py
View File

@@ -17,7 +17,10 @@ class TestAddedToken:
def test_instantiate_with_content_only(self):
added_token = AddedToken("<mask>")
added_token.content = "<MASK>"
assert added_token.content == "<MASK>"
assert type(added_token) == AddedToken
added_token.content = added_token.content.lower()
assert str(added_token) == "<mask>"
assert (
repr(added_token) == 'AddedToken("<mask>", rstrip=False, lstrip=False, single_word=False, normalized=True, special=False)'