mii/tokenizers
1
0
Fork 0
mirror of https://github.com/mii443/tokenizers.git synced 2025-08-22 16:25:30 +00:00
Code Issues Packages Projects Releases Wiki Activity

clippy (#1781)

Browse Source 
* clippy

* fmtr

* rutc?

* fix onig issue

* up

* decode stream default

* jump a release for cargo audit ...

* more cliippy stuff

* clippy?

* proper style

* fmt
This commit is contained in:
Arthur
2025-05-27 11:30:32 +02:00
committed by GitHub
parent 23e7e42adf
commit 01f8bc834c
18 changed files with 46 additions and 59 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
bindings/python/Cargo.toml
View File

@ -14,8 +14,8 @@ serde = { version = "1.0", features = ["rc", "derive"] }
serde_json = "1.0"
libc = "0.2"
env_logger = "0.11"
pyo3 = { version = "0.23", features = ["abi3", "abi3-py39", "py-clone"] }
numpy = "0.23"
pyo3 = { version = "0.24", features = ["abi3", "abi3-py39", "py-clone"] }
numpy = "0.24"
ndarray = "0.16"
itertools = "0.12"
@ -24,7 +24,7 @@ path = "../../tokenizers"
[dev-dependencies]
tempfile = "3.10"
pyo3 = { version = "0.23", features = ["auto-initialize"] }
pyo3 = { version = "0.24", features = ["auto-initialize"] }
[features]
default = ["pyo3/extension-module"]

2
bindings/python/py_src/tokenizers/decoders/__init__.pyi
View File

@ -33,7 +33,7 @@ class BPEDecoder(Decoder):
Args:
suffix (:obj:`str`, `optional`, defaults to :obj:`</w>`):
The suffix that was used to caracterize an end-of-word. This suffix will
The suffix that was used to characterize an end-of-word. This suffix will
be replaced by whitespaces during the decoding
"""
def __init__(self, suffix="</w>"):