mii/tokenizers
1
0
Fork 0
mirror of https://github.com/mii443/tokenizers.git synced 2025-08-22 16:25:30 +00:00
Code Issues Packages Projects Releases Wiki Activity

Fixing for clippy 1.78 (#1548)

Browse Source
This commit is contained in:
Nicolas Patry
2024-06-06 13:18:59 +02:00
committed by GitHub
parent bfefcf676d
commit 8d28dbefd1
2 changed files with 3 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
tokenizers/src/models/bpe/trainer.rs
View File

@ -523,7 +523,7 @@ impl BpeTrainer {
.get(&new_token)
.copied()
.unwrap_or(id_to_word.len() as u32);
if word_to_id.get(&new_token).is_none() {
if !word_to_id.contains_key(&new_token) {
id_to_word.push(new_token.clone());
word_to_id.insert(new_token.clone(), new_token_id);
}

4
tokenizers/src/models/wordpiece/mod.rs
View File

@ -180,10 +180,10 @@ impl WordPiece {
pub fn from_bpe(bpe: &BPE) -> Self {
let mut wp = Self::builder().vocab(bpe.get_vocab()).build().unwrap();
if let Some(unk) = bpe.get_unk_token() {
wp.unk_token = unk.to_owned();
unk.clone_into(&mut wp.unk_token);
}
if let Some(prefix) = bpe.get_continuing_subword_prefix() {
wp.continuing_subword_prefix = prefix.to_owned();
prefix.clone_into(&mut wp.continuing_subword_prefix);
}
wp
}