mii/tokenizers
1
0
Fork 0
mirror of https://github.com/mii443/tokenizers.git synced 2025-08-22 16:25:30 +00:00
Code Issues Packages Projects Releases Wiki Activity

Using clippy 1.67 (#1167)

Browse Source
This commit is contained in:
Nicolas Patry
2023-03-02 12:28:39 +01:00
committed by GitHub
parent ac552ff8b9
commit 3138657565
1 changed files with 1 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
tokenizers/src/tokenizer/added_vocabulary.rs
View File

@ -322,11 +322,7 @@ impl AddedVocabulary {
/// This method returns a list "splits", each of them being a pair of Offsets
/// and an optional ID if it is an AddedToken.
/// The list of splits cover the entire input string.
fn find_matches<'a>(
&self,
sentence: &str,
split_re: &'a MatchingSet,
) -> Vec<(Option<u32>, Offsets)> {
fn find_matches(&self, sentence: &str, split_re: &MatchingSet) -> Vec<(Option<u32>, Offsets)> {
if sentence.is_empty() {
return vec![(None, (0, 0))];
}