mii/tokenizers
1
0
Fork 0
mirror of https://github.com/mii443/tokenizers.git synced 2025-08-22 16:25:30 +00:00
Code Issues Packages Projects Releases Wiki Activity

correctly compute the new id: we take the max of the AddedToken + get_vocab_size

Browse Source
This commit is contained in:
Arthur Zucker
2023-09-01 19:03:33 +00:00
parent db319492f7
commit 2b72017e17
1 changed files with 1 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
tokenizers/src/tokenizer/added_vocabulary.rs
View File

@ -264,7 +264,7 @@ impl AddedVocabulary {
let id = if let Some(id) = self.token_to_id(&token.content, model) { let id = if let Some(id) = self.token_to_id(&token.content, model) {
id id
} else { } else {
let new_id = (model.get_vocab_size() + self.added_tokens_map.len()) as u32; let new_id = (model.get_vocab_size() + cmp::max(self.added_tokens_map_r.keys(),0)) as u32;
new_id new_id
}; };