mii/tokenizers
1
0
Fork 0
mirror of https://github.com/mii443/tokenizers.git synced 2025-12-03 11:18:29 +00:00
Code Issues Packages Projects Releases Wiki Activity

Revert "Fix SPM conversions"

Browse Source 
This reverts commit e1ffe39764.
This commit is contained in:
Lysandre
2021-04-21 18:09:36 -04:00
parent e1ffe39764
commit e999a7b5f9
1 changed files with 2 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
bindings/python/scripts/convert.py
View File

@@ -131,7 +131,7 @@ class AlbertConverter(SpmConverter):
]
def normalizer(self, proto):
normalizers = [Replace("``", '"'), Replace("''", '"')]
normalizers = [Replace("``", '"'), Replace("''", '"'), Replace(Regex(" {2,}"), " ")]
if not self.original_tokenizer.keep_accents:
normalizers.append(NFKD())
normalizers.append(StripAccents())
@@ -140,7 +140,6 @@ class AlbertConverter(SpmConverter):
precompiled_charsmap = proto.normalizer_spec.precompiled_charsmap
normalizers.append(Precompiled(precompiled_charsmap))
normalizers.append(Replace(Regex(" {2,}"), " "))
return Sequence(normalizers)
def post_processor(self, tokenizer):
@@ -268,7 +267,7 @@ class XLNetConverter(SpmConverter):
]
def normalizer(self, proto):
normalizers = [Replace("``", '"'), Replace("''", '"')]
normalizers = [Replace("``", '"'), Replace("''", '"'), Replace(Regex(" {2,}"), " ")]
if not self.original_tokenizer.keep_accents:
normalizers.append(NFKD())
normalizers.append(StripAccents())
@@ -277,7 +276,6 @@ class XLNetConverter(SpmConverter):
precompiled_charsmap = proto.normalizer_spec.precompiled_charsmap
normalizers.append(Precompiled(precompiled_charsmap))
normalizers.append(Replace(Regex(" {2,}"), " "))
return Sequence(normalizers)
def post_processor(self, tokenizer):