mirror of
https://github.com/mii443/tokenizers.git
synced 2025-12-05 12:18:20 +00:00
Python - Update PyNormalizer interface
This commit is contained in:
@@ -1,3 +1,4 @@
|
||||
from .. import NormalizedString
|
||||
from typing import Optional, List
|
||||
|
||||
class Normalizer:
|
||||
@@ -7,6 +8,13 @@ class Normalizer:
|
||||
Normalizer will return an instance of this class when instantiated.
|
||||
"""
|
||||
|
||||
def normalize(self, normalized: NormalizedString):
|
||||
""" Normalize the given NormalizedString in-place """
|
||||
pass
|
||||
def normalize_str(self, sequence: str) -> str:
|
||||
""" Normalize the given str """
|
||||
pass
|
||||
|
||||
class BertNormalizer(Normalizer):
|
||||
""" BertNormalizer
|
||||
|
||||
|
||||
Reference in New Issue
Block a user