mirror of
https://github.com/mii443/tokenizers.git
synced 2025-12-03 03:08:21 +00:00
Fix typos in strings and comments (#1770)
This commit is contained in:
@@ -27,4 +27,4 @@ tempfile = "3.10"
|
||||
pyo3 = { version = "0.23", features = ["auto-initialize"] }
|
||||
|
||||
[features]
|
||||
defaut = ["pyo3/extension-module"]
|
||||
default = ["pyo3/extension-module"]
|
||||
|
||||
@@ -397,7 +397,7 @@ def main():
|
||||
"--models",
|
||||
type=lambda s: s.split(","),
|
||||
default=pretraineds,
|
||||
help=f"The pretrained tokenizers you want to test agains, (default: {pretraineds})",
|
||||
help=f"The pretrained tokenizers you want to test against, (default: {pretraineds})",
|
||||
)
|
||||
args = parser.parse_args()
|
||||
|
||||
|
||||
@@ -404,7 +404,7 @@ impl PyMetaspaceDec {
|
||||
///
|
||||
/// Args:
|
||||
/// suffix (:obj:`str`, `optional`, defaults to :obj:`</w>`):
|
||||
/// The suffix that was used to caracterize an end-of-word. This suffix will
|
||||
/// The suffix that was used to characterize an end-of-word. This suffix will
|
||||
/// be replaced by whitespaces during the decoding
|
||||
#[pyclass(extends=PyDecoder, module = "tokenizers.decoders", name = "BPEDecoder")]
|
||||
pub struct PyBPEDecoder {}
|
||||
|
||||
Reference in New Issue
Block a user