mirror of
https://github.com/mii443/tokenizers.git
synced 2025-08-22 16:25:30 +00:00
Fix typos in strings and comments (#1770)
This commit is contained in:
@ -122,7 +122,7 @@ describe('Encoding', () => {
|
||||
expect(indexes).toEqual([3, 5])
|
||||
})
|
||||
|
||||
it('returns the corrent indexes with pair sequences', () => {
|
||||
it('returns the correct indexes with pair sequences', () => {
|
||||
expect(encodingDual.wordToTokens(3, 0)).toEqual([3, 5])
|
||||
expect(encodingDual.wordToTokens(3, 1)).toEqual([8, 9])
|
||||
})
|
||||
|
@ -27,4 +27,4 @@ tempfile = "3.10"
|
||||
pyo3 = { version = "0.23", features = ["auto-initialize"] }
|
||||
|
||||
[features]
|
||||
defaut = ["pyo3/extension-module"]
|
||||
default = ["pyo3/extension-module"]
|
||||
|
@ -397,7 +397,7 @@ def main():
|
||||
"--models",
|
||||
type=lambda s: s.split(","),
|
||||
default=pretraineds,
|
||||
help=f"The pretrained tokenizers you want to test agains, (default: {pretraineds})",
|
||||
help=f"The pretrained tokenizers you want to test against, (default: {pretraineds})",
|
||||
)
|
||||
args = parser.parse_args()
|
||||
|
||||
|
@ -404,7 +404,7 @@ impl PyMetaspaceDec {
|
||||
///
|
||||
/// Args:
|
||||
/// suffix (:obj:`str`, `optional`, defaults to :obj:`</w>`):
|
||||
/// The suffix that was used to caracterize an end-of-word. This suffix will
|
||||
/// The suffix that was used to characterize an end-of-word. This suffix will
|
||||
/// be replaced by whitespaces during the decoding
|
||||
#[pyclass(extends=PyDecoder, module = "tokenizers.decoders", name = "BPEDecoder")]
|
||||
pub struct PyBPEDecoder {}
|
||||
|
Reference in New Issue
Block a user