mirror of
https://github.com/mii443/tokenizers.git
synced 2025-08-22 16:25:30 +00:00
python stub.py
This commit is contained in:
@ -28,13 +28,14 @@ class AddedToken:
|
|||||||
normalized (:obj:`bool`, defaults to :obj:`True` with :meth:`~tokenizers.Tokenizer.add_tokens` and :obj:`False` with :meth:`~tokenizers.Tokenizer.add_special_tokens`):
|
normalized (:obj:`bool`, defaults to :obj:`True` with :meth:`~tokenizers.Tokenizer.add_tokens` and :obj:`False` with :meth:`~tokenizers.Tokenizer.add_special_tokens`):
|
||||||
Defines whether this token should match against the normalized version of the input
|
Defines whether this token should match against the normalized version of the input
|
||||||
text. For example, with the added token ``"yesterday"``, and a normalizer in charge of
|
text. For example, with the added token ``"yesterday"``, and a normalizer in charge of
|
||||||
lowercasing the text, the token could be extracted from the input ``"I saw a lion
|
lowercasing the text, the token could be extract from the input ``"I saw a lion
|
||||||
Yesterday"``.
|
Yesterday"``.
|
||||||
special (:obj:`bool`, defaults to :obj:`False` with :meth:`~tokenizers.Tokenizer.add_tokens` and :obj:`False` with :meth:`~tokenizers.Tokenizer.add_special_tokens`):
|
special (:obj:`bool`, defaults to :obj:`False` with :meth:`~tokenizers.Tokenizer.add_tokens` and :obj:`False` with :meth:`~tokenizers.Tokenizer.add_special_tokens`):
|
||||||
Defines whether this token should be skipped when decoding.
|
Defines whether this token should be skipped when decoding.
|
||||||
|
|
||||||
"""
|
"""
|
||||||
|
|
||||||
def __init__(self, content, single_word=False, lstrip=False, rstrip=False, normalized=True):
|
def __init__(self, content, single_word=False, lstrip=False, rstrip=False, normalized=True, special=False):
|
||||||
pass
|
pass
|
||||||
@property
|
@property
|
||||||
def content(self):
|
def content(self):
|
||||||
@ -65,7 +66,7 @@ class AddedToken:
|
|||||||
"""
|
"""
|
||||||
Get the value of the :obj:`single_word` option
|
Get the value of the :obj:`single_word` option
|
||||||
"""
|
"""
|
||||||
pass
|
pass
|
||||||
@property
|
@property
|
||||||
def special(self):
|
def special(self):
|
||||||
"""
|
"""
|
||||||
|
Reference in New Issue
Block a user