mii/tokenizers
1
0
Fork 0
mirror of https://github.com/mii443/tokenizers.git synced 2025-12-06 12:48:18 +00:00
Code Issues Packages Projects Releases Wiki Activity

Fix number of added tokens

Browse Source 
Related to #302
This commit is contained in:
Anthony MOI
2020-06-11 14:22:58 -04:00
parent 67b458b134
commit bb68ec3414
1 changed files with 1 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
tokenizers/src/tokenizer/mod.rs
View File

@@ -886,6 +886,7 @@ impl Tokenizer {
} }
let id = if let Some(id) = self.token_to_id(&token.content) { let id = if let Some(id) = self.token_to_id(&token.content) {
ignored += 1;
id id
} else { } else {
let new_id = (self.model.get_vocab_size() + self.added_tokens_map.len()) as u32; let new_id = (self.model.get_vocab_size() + self.added_tokens_map.len()) as u32;