mirror of
https://github.com/mii443/tokenizers.git
synced 2025-12-08 05:38:23 +00:00
Add an Encoding.sequences to allow masking
This commit is contained in:
@@ -327,6 +327,17 @@ class Encoding:
|
||||
"""
|
||||
pass
|
||||
@property
|
||||
def sequences(self) -> List[Optional[int]]:
|
||||
"""The generated sequence indices.
|
||||
|
||||
They represent the index of the input sequence associated to each token.
|
||||
The sequence id can be None if the token is not related to any input sequence,
|
||||
like for example with special tokens.
|
||||
|
||||
Returns:
|
||||
A :obj:`List` of :obj:`Optional[int]`: A list of optional sequence index.
|
||||
"""
|
||||
@property
|
||||
def type_ids(self) -> List[int]:
|
||||
"""The generated type IDs
|
||||
|
||||
|
||||
Reference in New Issue
Block a user