diff --git a/bindings/python/py_src/tokenizers/__init__.py b/bindings/python/py_src/tokenizers/__init__.py index 745a6a4c..e05bf164 100644 --- a/bindings/python/py_src/tokenizers/__init__.py +++ b/bindings/python/py_src/tokenizers/__init__.py @@ -1,4 +1,4 @@ -__version__ = "0.13.4.rc1" +__version__ = "0.13.4.rc2" from enum import Enum from typing import List, Tuple, Union diff --git a/bindings/python/setup.py b/bindings/python/setup.py index 4f270415..f6b4c902 100644 --- a/bindings/python/setup.py +++ b/bindings/python/setup.py @@ -9,7 +9,7 @@ extras["dev"] = extras["testing"] setup( name="tokenizers", - version="0.13.4.rc1", + version="0.13.4.rc2", description="Fast and Customizable Tokenizers", long_description=open("README.md", "r", encoding="utf-8").read(), long_description_content_type="text/markdown", diff --git a/tokenizers/src/tokenizer/mod.rs b/tokenizers/src/tokenizer/mod.rs index 2c4382f5..ed0ade5e 100644 --- a/tokenizers/src/tokenizer/mod.rs +++ b/tokenizers/src/tokenizer/mod.rs @@ -605,7 +605,7 @@ where if let Some(trunc_params) = &trunc { let n_added_tokens = self.get_n_added_tokens(false); let effective_max_length = trunc_params.max_length - n_added_tokens; - if effective_max_length <= trunc_params.stride { + if effective_max_length < trunc_params.stride { return Err(Box::new(TruncationParamError(format!( "tokenizer stride set to {}, which is greater than or equal to its effective max length of {} (= {} original max length - {} added special tokens), ", trunc_params.stride, effective_max_length, trunc_params.max_length, n_added_tokens