mii/tokenizers
1
0
Fork 0
mirror of https://github.com/mii443/tokenizers.git synced 2025-08-23 00:35:35 +00:00
Code Issues Packages Projects Releases Wiki Activity

BertNormalizer has same behavior than original implem

Browse Source
This commit is contained in:
Anthony MOI
2020-07-06 13:32:19 -04:00
parent b91deeaa3d
commit 7a95ffc4fa
6 changed files with 13 additions and 11 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
bindings/python/src/normalizers.rs
View File

@ -51,7 +51,7 @@ impl BertNormalizer {
fn new(kwargs: Option<&PyDict>) -> PyResult<(Self, Normalizer)> {
let mut clean_text = true;
let mut handle_chinese_chars = true;
let mut strip_accents = true;
let mut strip_accents = None;
let mut lowercase = true;
if let Some(kwargs) = kwargs {