mii/tokenizers
1
0
Fork 0
mirror of https://github.com/mii443/tokenizers.git synced 2025-08-22 16:25:30 +00:00
Code Issues Packages Projects Releases Wiki Activity

Cannot add new tokens that already exist in the vocab

Browse Source
This commit is contained in:
Anthony MOI
2019-12-19 18:32:03 -05:00
parent 9763282d59
commit 076ba297fb
1 changed files with 1 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
tokenizers/src/tokenizer/mod.rs
View File

@ -471,7 +471,7 @@ impl Tokenizer {
pub fn add_tokens(&mut self, tokens: &[AddedToken]) -> usize { pub fn add_tokens(&mut self, tokens: &[AddedToken]) -> usize {
let mut ignored = 0; let mut ignored = 0;
for token in tokens { for token in tokens {
if token.content.is_empty() { if token.content.is_empty() || self.token_to_id(&token.content).is_some() {
ignored += 1; ignored += 1;
continue; continue;
} }