Fix number of added tokens

Related to #302
This commit is contained in:
Anthony MOI
2020-06-11 14:22:58 -04:00
parent 67b458b134
commit bb68ec3414

View File

@@ -886,6 +886,7 @@ impl Tokenizer {
}
let id = if let Some(id) = self.token_to_id(&token.content) {
ignored += 1;
id
} else {
let new_id = (self.model.get_vocab_size() + self.added_tokens_map.len()) as u32;