mirror of
https://github.com/mii443/tokenizers.git
synced 2025-08-22 16:25:30 +00:00
Add a way to specify the unknown token in SentencePieceUnigramTokenizer
python implem (#762)
* add a way to specify the unknown token in `SentencePieceUnigramTokenizer` * add test that verify that an exception is raised for the missing unknown token * style * add test tokens
This commit is contained in:
@ -2,7 +2,7 @@ from setuptools import setup
|
||||
from setuptools_rust import Binding, RustExtension
|
||||
|
||||
extras = {}
|
||||
extras["testing"] = ["pytest"]
|
||||
extras["testing"] = ["pytest", "requests", "numpy", "datasets"]
|
||||
|
||||
setup(
|
||||
name="tokenizers",
|
||||
|
Reference in New Issue
Block a user