mii/tokenizers
1
0
Fork 0
mirror of https://github.com/mii443/tokenizers.git synced 2025-08-23 00:35:35 +00:00
Code Issues Packages Projects Releases Wiki Activity

Fix BertProcessing overflowing usize

Browse Source
This commit is contained in:
Anthony MOI
2019-12-16 18:46:58 -05:00
parent 93a74aa53a
commit e4ce050b73
1 changed files with 5 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
tokenizers/src/processors/bert.rs
View File

@ -38,8 +38,11 @@ impl PostProcessor for BertProcessing {
.unwrap_or(0)
+ special_token_len;
let need_trunc = total_len - self.max_len;
println!("Need to trunc {}", need_trunc);
let need_trunc = if total_len > self.max_len {
total_len - self.max_len
} else {
0
};
let (mut encoding, pair_encoding) = truncate_encodings(
encoding,
pair_encoding,