mii/tokenizers
1
0
Fork 0
mirror of https://github.com/mii443/tokenizers.git synced 2025-08-23 16:49:27 +00:00
Code Issues Packages Projects Releases Wiki Activity

Fix BertProcessing overflowing usize

Browse Source
This commit is contained in:
Anthony MOI
2019-12-16 18:46:58 -05:00
parent 93a74aa53a
commit e4ce050b73
1 changed files with 5 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
tokenizers/src/processors/bert.rs
View File

@ -38,8 +38,11 @@ impl PostProcessor for BertProcessing {
.unwrap_or(0) .unwrap_or(0)
+ special_token_len; + special_token_len;
let need_trunc = total_len - self.max_len; let need_trunc = if total_len > self.max_len {
println!("Need to trunc {}", need_trunc); total_len - self.max_len
} else {
0
};
let (mut encoding, pair_encoding) = truncate_encodings( let (mut encoding, pair_encoding) = truncate_encodings(
encoding, encoding,
pair_encoding, pair_encoding,