mii/tokenizers
1
0
Fork 0
mirror of https://github.com/mii443/tokenizers.git synced 2025-08-23 00:35:35 +00:00
Code Issues Packages Projects Releases Wiki Activity

Fix truncate

Browse Source
This commit is contained in:
Anthony MOI
2019-12-23 17:27:43 -05:00
parent 8fb94be3d0
commit 2159123d7c
1 changed files with 1 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
tokenizers/src/tokenizer/encoding.rs
View File

@ -85,7 +85,7 @@ impl Encoding {
}
pub fn truncate(&mut self, max_len: usize, stride: usize) {
if max_len > self.ids.len() {
if max_len >= self.ids.len() {
return;
}