mirror of
https://github.com/mii443/tokenizers.git
synced 2025-08-23 00:35:35 +00:00
0d7c232f95
This allows testing versions not built in-place. Otherwise importing (or testing) in the package root fails without develop builds. Replace maturin with setuptools_rust since maturin fails with proper project structure.
12 lines
248 B
Python
12 lines
248 B
Python
from typing import List, Tuple
|
|
|
|
from .. import models, Offsets
|
|
|
|
TokenizedSequence = List[str]
|
|
TokenizedSequenceWithOffsets = List[Tuple[str, Offsets]]
|
|
|
|
Model = models.Model
|
|
BPE = models.BPE
|
|
WordPiece = models.WordPiece
|
|
WordLevel = models.WordLevel
|