mii/tokenizers
1
0
Fork 0
mirror of https://github.com/mii443/tokenizers.git synced 2025-08-22 16:25:30 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix a couple linting warnings

Browse Source
This commit is contained in:
epwalsh
2019-12-13 13:09:25 -08:00
committed by MOI Anthony
parent 1c4593cad4
commit dc48cc3680
2 changed files with 7 additions and 11 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
tokenizers/src/models/bpe/trainer.rs
View File

@ -158,12 +158,10 @@ impl Trainer for BpeTrainer {
let mut change_count = |pair: Pair, count: i32, word_index: usize| { let mut change_count = |pair: Pair, count: i32, word_index: usize| {
if pair_counts.contains_key(&pair) { if pair_counts.contains_key(&pair) {
pair_counts.get_mut(&pair).unwrap().0 += count; pair_counts.get_mut(&pair).unwrap().0 += count;
} else { } else if count > 0 {
if count > 0 { pair_counts.insert(pair, (count, pair));
pair_counts.insert(pair, (count, pair)); if !where_to_update.contains_key(&pair) {
if !where_to_update.contains_key(&pair) { where_to_update.insert(pair, HashSet::new());
where_to_update.insert(pair, HashSet::new());
}
} }
} }

8
tokenizers/src/utils.rs
View File

@ -59,12 +59,10 @@ pub fn truncate_encodings(
TruncationStrategy::OnlyFirst | TruncationStrategy::OnlySecond => { TruncationStrategy::OnlyFirst | TruncationStrategy::OnlySecond => {
let target = if strategy == TruncationStrategy::OnlyFirst { let target = if strategy == TruncationStrategy::OnlyFirst {
Ok(&mut encoding) Ok(&mut encoding)
} else if let Some(encoding) = pair_encoding.as_mut() {
Ok(encoding)
} else { } else {
if let Some(encoding) = pair_encoding.as_mut() { Err(Box::new(Error::SecondSequenceNotProvided))
Ok(encoding)
} else {
Err(Box::new(Error::SecondSequenceNotProvided))
}
}?; }?;
if target.get_ids().len() <= to_remove { if target.get_ids().len() <= to_remove {