make tests happy

2025-08-23 00:35:35 +00:00 · 2023-09-05 15:37:09 +00:00
parent 531b06f6db
commit 08af8ea9c3
7 changed files with 15 additions and 14 deletions
--- a/bindings/python/src/tokenizer.rs
+++ b/bindings/python/src/tokenizer.rs
@ -75,7 +75,7 @@ impl PyAddedToken {
            single_word: None,
            lstrip: None,
            rstrip: None,
-            normalized: None,
+            normalized: Some(!special.unwrap_or(true)),
        }
    }