mirror of
https://github.com/mii443/tokenizers.git
synced 2025-08-22 16:25:30 +00:00
cliipy
This commit is contained in:
@ -20,10 +20,11 @@ class TestAddedToken:
|
|||||||
assert added_token.content == "<MASK>"
|
assert added_token.content == "<MASK>"
|
||||||
assert type(added_token) == AddedToken
|
assert type(added_token) == AddedToken
|
||||||
added_token.content = added_token.content.lower()
|
added_token.content = added_token.content.lower()
|
||||||
|
|
||||||
assert str(added_token) == "<mask>"
|
assert str(added_token) == "<mask>"
|
||||||
assert (
|
assert (
|
||||||
repr(added_token) == 'AddedToken("<mask>", rstrip=False, lstrip=False, single_word=False, normalized=True, special=False)'
|
repr(added_token)
|
||||||
|
== 'AddedToken("<mask>", rstrip=False, lstrip=False, single_word=False, normalized=True, special=False)'
|
||||||
)
|
)
|
||||||
assert added_token.rstrip == False
|
assert added_token.rstrip == False
|
||||||
assert added_token.lstrip == False
|
assert added_token.lstrip == False
|
||||||
|
Reference in New Issue
Block a user