python stub.py

This commit is contained in:
Arthur Zucker
2023-09-04 19:49:36 +00:00
parent b235f85527
commit 2291c89896

View File

@ -28,13 +28,14 @@ class AddedToken:
normalized (:obj:`bool`, defaults to :obj:`True` with :meth:`~tokenizers.Tokenizer.add_tokens` and :obj:`False` with :meth:`~tokenizers.Tokenizer.add_special_tokens`): normalized (:obj:`bool`, defaults to :obj:`True` with :meth:`~tokenizers.Tokenizer.add_tokens` and :obj:`False` with :meth:`~tokenizers.Tokenizer.add_special_tokens`):
Defines whether this token should match against the normalized version of the input Defines whether this token should match against the normalized version of the input
text. For example, with the added token ``"yesterday"``, and a normalizer in charge of text. For example, with the added token ``"yesterday"``, and a normalizer in charge of
lowercasing the text, the token could be extracted from the input ``"I saw a lion lowercasing the text, the token could be extract from the input ``"I saw a lion
Yesterday"``. Yesterday"``.
special (:obj:`bool`, defaults to :obj:`False` with :meth:`~tokenizers.Tokenizer.add_tokens` and :obj:`False` with :meth:`~tokenizers.Tokenizer.add_special_tokens`): special (:obj:`bool`, defaults to :obj:`False` with :meth:`~tokenizers.Tokenizer.add_tokens` and :obj:`False` with :meth:`~tokenizers.Tokenizer.add_special_tokens`):
Defines whether this token should be skipped when decoding. Defines whether this token should be skipped when decoding.
""" """
def __init__(self, content, single_word=False, lstrip=False, rstrip=False, normalized=True): def __init__(self, content, single_word=False, lstrip=False, rstrip=False, normalized=True, special=False):
pass pass
@property @property
def content(self): def content(self):
@ -65,7 +66,7 @@ class AddedToken:
""" """
Get the value of the :obj:`single_word` option Get the value of the :obj:`single_word` option
""" """
pass pass
@property @property
def special(self): def special(self):
""" """