mii/tokenizers
1
0
Fork 0
mirror of https://github.com/mii443/tokenizers.git synced 2025-08-22 16:25:30 +00:00
Code Issues Packages Projects Releases Wiki Activity

Fixing NormalizedString append when normalized is empty. (#1717)

Browse Source 
Co-authored-by: Anantha Kandrapu <anantkan@amazon.com>
This commit is contained in:
Nicolas Patry
2025-01-09 17:41:32 +01:00
committed by GitHub
parent 0ff2ab0f64
commit e7ed39de3c
1 changed files with 23 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

23
tokenizers/src/tokenizer/normalizer.rs
View File

@ -517,6 +517,9 @@ impl NormalizedString {
if let Some((b, prev)) = self.normalized.char_indices().last() { if let Some((b, prev)) = self.normalized.char_indices().last() {
let transformations = std::iter::once((prev, 0)).chain(s.chars().map(|c| (c, 1))); let transformations = std::iter::once((prev, 0)).chain(s.chars().map(|c| (c, 1)));
self.transform_range(Range::Normalized(b..), transformations, 0); self.transform_range(Range::Normalized(b..), transformations, 0);
} else {
let transformations = s.chars().map(|c| (c, 1));
self.transform_range(Range::Normalized(..), transformations, 0);
} }
self self
} }
@ -2284,4 +2287,24 @@ mod tests {
s.lowercase(); s.lowercase();
assert_eq!(s.get(), "a..."); assert_eq!(s.get(), "a...");
} }
#[test]
fn test_append_after_clear() {
let mut n = NormalizedString::from("Hello");
assert_eq!(n.get(), "Hello");
n.clear();
assert_eq!(n.get(), "");
n.append(" World");
assert_eq!(n.get(), " World");
assert_eq!(n.len_original(), 5);
assert_eq!(n.len(), 6);
assert_eq!(n.get_range_original(Range::Original(0..5)), Some("Hello"));
assert_eq!(n.get_range_original(Range::Normalized(0..6)), Some(""));
assert_eq!(n.get_range(Range::Normalized(0..6)), Some(" World"));
}
} }