mirror of
https://github.com/mii443/tokenizers.git
synced 2025-12-18 06:19:14 +00:00
Python - PyBertNormalizer can get/set its attributes
This commit is contained in:
@@ -44,6 +44,26 @@ class TestBertNormalizer:
|
||||
output = normalizer.normalize_str("Héllò")
|
||||
assert output == "héllò"
|
||||
|
||||
def test_can_modify(self):
|
||||
normalizer = BertNormalizer(
|
||||
clean_text=True, handle_chinese_chars=True, strip_accents=True, lowercase=True
|
||||
)
|
||||
|
||||
assert normalizer.clean_text == True
|
||||
assert normalizer.handle_chinese_chars == True
|
||||
assert normalizer.strip_accents == True
|
||||
assert normalizer.lowercase == True
|
||||
|
||||
# Modify these
|
||||
normalizer.clean_text = False
|
||||
assert normalizer.clean_text == False
|
||||
normalizer.handle_chinese_chars = False
|
||||
assert normalizer.handle_chinese_chars == False
|
||||
normalizer.strip_accents = None
|
||||
assert normalizer.strip_accents == None
|
||||
normalizer.lowercase = False
|
||||
assert normalizer.lowercase == False
|
||||
|
||||
|
||||
class TestSequence:
|
||||
def test_instantiate(self):
|
||||
|
||||
Reference in New Issue
Block a user