mirror of
https://github.com/mii443/tokenizers.git
synced 2025-12-16 17:18:43 +00:00
TokenizedSequence / TokenizedSequenceWithOffsets needs to be declared in .py files not only .pyi
This commit is contained in:
committed by
Anthony MOI
parent
68405a6fae
commit
39958a2f0f
@@ -1,5 +1,8 @@
|
||||
__version__ = "0.6.0"
|
||||
|
||||
from typing import Tuple
|
||||
Offsets = Tuple[int, int]
|
||||
|
||||
from .tokenizers import Tokenizer, Encoding, AddedToken
|
||||
from .tokenizers import decoders
|
||||
from .tokenizers import models
|
||||
|
||||
@@ -1,4 +1,9 @@
|
||||
from .. import models
|
||||
from typing import List, Tuple
|
||||
|
||||
from .. import models, Offsets
|
||||
|
||||
TokenizedSequence = List[str]
|
||||
TokenizedSequenceWithOffsets = List[Tuple[str, Offsets]]
|
||||
|
||||
Model = models.Model
|
||||
BPE = models.BPE
|
||||
|
||||
Reference in New Issue
Block a user