mirror of
https://github.com/mii443/tokenizers.git
synced 2025-08-23 00:35:35 +00:00
0d7c232f95
This allows testing versions not built in-place. Otherwise importing (or testing) in the package root fails without develop builds. Replace maturin with setuptools_rust since maturin fails with proper project structure.
8 lines
178 B
Python
8 lines
178 B
Python
from .. import decoders
|
|
|
|
Decoder = decoders.Decoder
|
|
ByteLevel = decoders.ByteLevel
|
|
WordPiece = decoders.WordPiece
|
|
Metaspace = decoders.Metaspace
|
|
BPEDecoder = decoders.BPEDecoder
|