mii/tokenizers
1
0
Fork 0
mirror of https://github.com/mii443/tokenizers.git synced 2025-12-05 20:28:22 +00:00
Code Issues Packages Projects Releases Wiki Activity

Python - Add bindings for NormalizedString

Browse Source
This commit is contained in:
Anthony MOI
2020-09-18 14:52:58 -04:00
committed by Anthony MOI
parent e4b10e0fd9
commit fce6998dcf
9 changed files with 537 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
bindings/python/py_src/tokenizers/__init__.py
View File

@@ -14,7 +14,7 @@ PreTokenizedEncodeInput = Union[
InputSequence = Union[TextInputSequence, PreTokenizedInputSequence]
EncodeInput = Union[TextEncodeInput, PreTokenizedEncodeInput]
from .tokenizers import Tokenizer, Encoding, AddedToken
from .tokenizers import Tokenizer, Encoding, AddedToken, Regex, NormalizedString
from .tokenizers import decoders
from .tokenizers import models
from .tokenizers import normalizers

7
bindings/python/py_src/tokenizers/__init__.pyi
View File

@@ -26,6 +26,13 @@ PreTokenizedEncodeInput = Union[
InputSequence = Union[TextInputSequence, PreTokenizedInputSequence]
EncodeInput = Union[TextEncodeInput, PreTokenizedEncodeInput]
class Regex:
""" A Regex """
def __new__(pattern: str) -> Regex:
""" Instantiate a new Regex with the given pattern """
pass
class Encoding:
""" An Encoding as returned by the Tokenizer """