mii/tokenizers
1
0
Fork 0
mirror of https://github.com/mii443/tokenizers.git synced 2025-08-22 16:25:30 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #93 from Tomarchelone/patch-1

Browse Source 
Fix indexing bug in add_tokens()
This commit is contained in:
MOI Anthony
2020-01-22 15:47:33 -05:00
committed by GitHub
parent b059190259 048ab46089
commit 36cd67fdf5
1 changed files with 1 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
tokenizers/src/tokenizer/mod.rs
View File

@ -587,7 +587,7 @@ impl Tokenizer {
continue;
}
let new_id = (self.model.get_vocab_size() - 1 + self.added_tokens.len()) as u32;
let new_id = (self.model.get_vocab_size() + self.added_tokens.len()) as u32;
let id = self
.added_tokens
.entry(token.clone())