mii/tokenizers
1
0
Fork 0
mirror of https://github.com/mii443/tokenizers.git synced 2025-08-23 00:35:35 +00:00
Code Issues Packages Projects Releases Wiki Activity

Fixing for clippy 1.78 (#1548)

Browse Source
This commit is contained in:
Nicolas Patry
2024-06-06 13:18:59 +02:00
committed by GitHub
parent bfefcf676d
commit 8d28dbefd1
2 changed files with 3 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
tokenizers/src/models/bpe/trainer.rs
View File

@ -523,7 +523,7 @@ impl BpeTrainer {
.get(&new_token) .get(&new_token)
.copied() .copied()
.unwrap_or(id_to_word.len() as u32); .unwrap_or(id_to_word.len() as u32);
if word_to_id.get(&new_token).is_none() { if !word_to_id.contains_key(&new_token) {
id_to_word.push(new_token.clone()); id_to_word.push(new_token.clone());
word_to_id.insert(new_token.clone(), new_token_id); word_to_id.insert(new_token.clone(), new_token_id);
} }

4
tokenizers/src/models/wordpiece/mod.rs
View File

@ -180,10 +180,10 @@ impl WordPiece {
pub fn from_bpe(bpe: &BPE) -> Self { pub fn from_bpe(bpe: &BPE) -> Self {
let mut wp = Self::builder().vocab(bpe.get_vocab()).build().unwrap(); let mut wp = Self::builder().vocab(bpe.get_vocab()).build().unwrap();
if let Some(unk) = bpe.get_unk_token() { if let Some(unk) = bpe.get_unk_token() {
wp.unk_token = unk.to_owned(); unk.clone_into(&mut wp.unk_token);
} }
if let Some(prefix) = bpe.get_continuing_subword_prefix() { if let Some(prefix) = bpe.get_continuing_subword_prefix() {
wp.continuing_subword_prefix = prefix.to_owned(); prefix.clone_into(&mut wp.continuing_subword_prefix);
} }
wp wp
} }