mii/tokenizers
1
0
Fork 0
mirror of https://github.com/mii443/tokenizers.git synced 2025-12-03 11:18:29 +00:00
Code Issues Packages Projects Releases Wiki Activity

nits

Browse Source
This commit is contained in:
Arthur Zucker
2023-09-05 15:40:13 +00:00
parent 08af8ea9c3
commit c3fa75fa0e
1 changed files with 1 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
bindings/python/src/tokenizer.rs
View File

@@ -75,7 +75,7 @@ impl PyAddedToken {
single_word: None, single_word: None,
lstrip: None, lstrip: None,
rstrip: None, rstrip: None,
normalized: Some(!special.unwrap_or(true)), normalized: None,
} }
} }