mii/tokenizers
1
0
Fork 0
mirror of https://github.com/mii443/tokenizers.git synced 2025-08-22 16:25:30 +00:00
Code Issues Packages Projects Releases Wiki Activity

python stub.py

Browse Source
This commit is contained in:
Arthur Zucker
2023-09-04 19:49:36 +00:00
parent b235f85527
commit 2291c89896
1 changed files with 5 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
bindings/python/py_src/tokenizers/__init__.pyi
View File

@ -28,13 +28,14 @@ class AddedToken:
normalized (:obj:`bool`, defaults to :obj:`True` with :meth:`~tokenizers.Tokenizer.add_tokens` and :obj:`False` with :meth:`~tokenizers.Tokenizer.add_special_tokens`):
Defines whether this token should match against the normalized version of the input
text. For example, with the added token ``"yesterday"``, and a normalizer in charge of
lowercasing the text, the token could be extracted from the input ``"I saw a lion
lowercasing the text, the token could be extract from the input ``"I saw a lion
Yesterday"``.
special (:obj:`bool`, defaults to :obj:`False` with :meth:`~tokenizers.Tokenizer.add_tokens` and :obj:`False` with :meth:`~tokenizers.Tokenizer.add_special_tokens`):
Defines whether this token should be skipped when decoding.
"""
def __init__(self, content, single_word=False, lstrip=False, rstrip=False, normalized=True):
def __init__(self, content, single_word=False, lstrip=False, rstrip=False, normalized=True, special=False):
pass
@property
def content(self):
@ -65,7 +66,7 @@ class AddedToken:
"""
Get the value of the :obj:`single_word` option
"""
pass
pass
@property
def special(self):
"""