diff --git a/tokenizers/src/processors/bert.rs b/tokenizers/src/processors/bert.rs index 0464c97d..e67c3cc6 100644 --- a/tokenizers/src/processors/bert.rs +++ b/tokenizers/src/processors/bert.rs @@ -38,8 +38,11 @@ impl PostProcessor for BertProcessing { .unwrap_or(0) + special_token_len; - let need_trunc = total_len - self.max_len; - println!("Need to trunc {}", need_trunc); + let need_trunc = if total_len > self.max_len { + total_len - self.max_len + } else { + 0 + }; let (mut encoding, pair_encoding) = truncate_encodings( encoding, pair_encoding,