mirror of
https://github.com/mii443/tokenizers.git
synced 2025-08-22 16:25:30 +00:00
fbe3365a13
* Update metadata as python3.7 and python3.8 support was dropped * Format pyproject.toml: unify quotes and indentation
70 lines
1.9 KiB
TOML
70 lines
1.9 KiB
TOML
[project]
|
|
name = "tokenizers"
|
|
requires-python = ">=3.9"
|
|
authors = [
|
|
{ name = "Nicolas Patry", email = "patry.nicolas@protonmail.com" },
|
|
{ name = "Anthony Moi", email = "anthony@huggingface.co" },
|
|
]
|
|
classifiers = [
|
|
"Development Status :: 5 - Production/Stable",
|
|
"Intended Audience :: Developers",
|
|
"Intended Audience :: Education",
|
|
"Intended Audience :: Science/Research",
|
|
"License :: OSI Approved :: Apache Software License",
|
|
"Operating System :: OS Independent",
|
|
"Programming Language :: Python :: 3",
|
|
"Programming Language :: Python :: 3.9",
|
|
"Programming Language :: Python :: 3.10",
|
|
"Programming Language :: Python :: 3.11",
|
|
"Programming Language :: Python :: 3.12",
|
|
"Programming Language :: Python :: 3.13",
|
|
"Programming Language :: Python :: 3 :: Only",
|
|
"Topic :: Scientific/Engineering :: Artificial Intelligence",
|
|
]
|
|
keywords = ["NLP", "tokenizer", "BPE", "transformer", "deep learning"]
|
|
dynamic = ["description", "license", "readme", "version"]
|
|
dependencies = ["huggingface_hub>=0.16.4,<1.0"]
|
|
|
|
[project.urls]
|
|
Homepage = "https://github.com/huggingface/tokenizers"
|
|
Source = "https://github.com/huggingface/tokenizers"
|
|
|
|
|
|
[project.optional-dependencies]
|
|
testing = ["pytest", "requests", "numpy", "datasets", "black==22.3", "ruff"]
|
|
docs = ["sphinx", "sphinx_rtd_theme", "setuptools_rust"]
|
|
dev = ["tokenizers[testing]"]
|
|
|
|
|
|
[build-system]
|
|
requires = ["maturin>=1.0,<2.0"]
|
|
build-backend = "maturin"
|
|
|
|
[tool.maturin]
|
|
python-source = "py_src"
|
|
module-name = "tokenizers.tokenizers"
|
|
bindings = "pyo3"
|
|
features = ["pyo3/extension-module"]
|
|
|
|
[tool.black]
|
|
line-length = 119
|
|
target-version = ["py35"]
|
|
|
|
[tool.ruff]
|
|
line-length = 119
|
|
target-version = "py311"
|
|
lint.ignore = [
|
|
# a == None in tests vs is None.
|
|
"E711",
|
|
# a == False in tests vs is False.
|
|
"E712",
|
|
# try.. import except.. pattern without using the lib.
|
|
"F401",
|
|
# Raw type equality is required in asserts
|
|
"E721",
|
|
# Import order
|
|
"E402",
|
|
# Fixtures unused import
|
|
"F811",
|
|
]
|