mirror of
https://github.com/mii443/tokenizers.git
synced 2025-08-23 16:49:27 +00:00
nits
This commit is contained in:
@ -75,7 +75,7 @@ impl PyAddedToken {
|
|||||||
single_word: None,
|
single_word: None,
|
||||||
lstrip: None,
|
lstrip: None,
|
||||||
rstrip: None,
|
rstrip: None,
|
||||||
normalized: Some(!special.unwrap_or(true)),
|
normalized: None,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
Reference in New Issue
Block a user