mii/tokenizers
1
0
Fork 0
mirror of https://github.com/mii443/tokenizers.git synced 2025-08-22 16:25:30 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix a couple linting warnings

Browse Source
This commit is contained in:
epwalsh
2019-12-13 13:09:25 -08:00
committed by MOI Anthony
parent 1c4593cad4
commit dc48cc3680
2 changed files with 7 additions and 11 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
tokenizers/src/models/bpe/trainer.rs
View File

@ -158,14 +158,12 @@ impl Trainer for BpeTrainer {
let mut change_count = |pair: Pair, count: i32, word_index: usize| {
if pair_counts.contains_key(&pair) {
pair_counts.get_mut(&pair).unwrap().0 += count;
} else {
if count > 0 {
} else if count > 0 {
pair_counts.insert(pair, (count, pair));
if !where_to_update.contains_key(&pair) {
where_to_update.insert(pair, HashSet::new());
}
}
}
if count > 0 {
where_to_update.get_mut(&pair).unwrap().insert(word_index);

4
tokenizers/src/utils.rs
View File

@ -59,12 +59,10 @@ pub fn truncate_encodings(
TruncationStrategy::OnlyFirst | TruncationStrategy::OnlySecond => {
let target = if strategy == TruncationStrategy::OnlyFirst {
Ok(&mut encoding)
} else {
if let Some(encoding) = pair_encoding.as_mut() {
} else if let Some(encoding) = pair_encoding.as_mut() {
Ok(encoding)
} else {
Err(Box::new(Error::SecondSequenceNotProvided))
}
}?;
if target.get_ids().len() <= to_remove {