mii/tokenizers
1
0
Fork 0
mirror of https://github.com/mii443/tokenizers.git synced 2025-08-23 00:35:35 +00:00
Code Issues Packages Projects Releases Wiki Activity

Remove comments. (#925)

Browse Source 
They are indeed a bit misleading and coupled with the REGEXP anyway.
This commit is contained in:
Nicolas Patry
2022-02-25 16:38:01 +01:00
committed by GitHub
parent d98abc50ba
commit b4c3844ffb
1 changed files with 0 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
tokenizers/src/tokenizer/added_vocabulary.rs
View File

@ -110,10 +110,6 @@ fn space_leftmost_at_end(sentence: &str) -> usize {
if let Some(match_) = LEFTMOST_SPACE_AT_END.find(sentence) {
match_.start()
} else {
// This should never happen since the Regex should match all the time
// It didn't match any space, so we can safely return sentence.len() to be the leftmot space and the
// end
warn!("Warning in leftmost_at_end code path, we expect the regex to match always and it didn't, Please report this.");
sentence.len()
}
}
@ -121,10 +117,6 @@ fn space_rightmost_at_start(sentence: &str) -> usize {
if let Some(match_) = RIGHTMOST_SPACE_AT_START.find(sentence) {
match_.end()
} else {
// This should never happen since the Regex should match all the time
// It didn't match any space, so we can safely return 0 to be the rightmost space and the
// start
warn!("Warning in leftmost_at_end code path, we expect the regex to match always and it didn't, Please report this.");
0
}
}