mii/tokenizers
1
0
Fork 0
mirror of https://github.com/mii443/tokenizers.git synced 2025-08-22 16:25:30 +00:00
Code Issues Packages Projects Releases Wiki Activity

Fix stride condition. (#1321)

Browse Source 
* Release all at once for simplicity.

* rc2
This commit is contained in:
Nicolas Patry
2023-08-14 15:27:55 +02:00
committed by GitHub
parent b35d33f981
commit 9a93c50c25
3 changed files with 3 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
bindings/python/py_src/tokenizers/__init__.py
View File

@ -1,4 +1,4 @@
__version__ = "0.13.4.rc1" __version__ = "0.13.4.rc2"
from enum import Enum from enum import Enum
from typing import List, Tuple, Union from typing import List, Tuple, Union

2
bindings/python/setup.py
View File

@ -9,7 +9,7 @@ extras["dev"] = extras["testing"]
setup( setup(
name="tokenizers", name="tokenizers",
version="0.13.4.rc1", version="0.13.4.rc2",
description="Fast and Customizable Tokenizers", description="Fast and Customizable Tokenizers",
long_description=open("README.md", "r", encoding="utf-8").read(), long_description=open("README.md", "r", encoding="utf-8").read(),
long_description_content_type="text/markdown", long_description_content_type="text/markdown",

2
tokenizers/src/tokenizer/mod.rs
View File

@ -605,7 +605,7 @@ where
if let Some(trunc_params) = &trunc { if let Some(trunc_params) = &trunc {
let n_added_tokens = self.get_n_added_tokens(false); let n_added_tokens = self.get_n_added_tokens(false);
let effective_max_length = trunc_params.max_length - n_added_tokens; let effective_max_length = trunc_params.max_length - n_added_tokens;
if effective_max_length <= trunc_params.stride { if effective_max_length < trunc_params.stride {
return Err(Box::new(TruncationParamError(format!( return Err(Box::new(TruncationParamError(format!(
"tokenizer stride set to {}, which is greater than or equal to its effective max length of {} (= {} original max length - {} added special tokens), ", "tokenizer stride set to {}, which is greater than or equal to its effective max length of {} (= {} original max length - {} added special tokens), ",
trunc_params.stride, effective_max_length, trunc_params.max_length, n_added_tokens trunc_params.stride, effective_max_length, trunc_params.max_length, n_added_tokens