Implement suggestions by @sebpuetz

Co-authored-by: Sebastian Pütz <sebastian.puetz@uni-tuebingen.de>
2025-12-05 20:28:22 +00:00 · 2020-07-31 16:49:00 -04:00
parent efb29582c6
commit dad70e8e85
11 changed files with 161 additions and 157 deletions
--- a/bindings/python/src/pre_tokenizers.rs
+++ b/bindings/python/src/pre_tokenizers.rs
@@ -61,7 +61,7 @@ impl PreTokenizer {
        .into_py()?;

        Ok(pretokenized
-            .get_normalized(true)
+            .get_normalized(tk::OffsetReferential::Original)
            .into_iter()
            .map(|(s, o)| (s.to_owned(), o))
            .collect())