mii/tokenizers
1
0
Fork 0
mirror of https://github.com/mii443/tokenizers.git synced 2025-08-22 16:25:30 +00:00
Code Issues Packages Projects Releases Wiki Activity

Updating convert scripts with Replace normalizer.

Browse Source
This commit is contained in:
Nicolas Patry
2020-09-18 12:29:13 +02:00
parent c59b216baa
commit 033b98ce59
1 changed files with 3 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
bindings/python/scripts/convert.py
View File

@ -11,6 +11,7 @@ from tokenizers.normalizers import (
Sequence,
BertNormalizer,
Precompiled,
Replace,
)
from tokenizers.pre_tokenizers import (
Digits,
@ -162,8 +163,7 @@ class AlbertConverter(SpmConverter):
]
def normalizer(self, proto):
normalizers = []
# TODO Missing Replace quotes
normalizers = [Replace("``", '"'), Replace("''", '"')]
if not self.original_tokenizer.keep_accents:
normalizers.append(NFKD())
normalizers.append(StripAccents())
@ -299,8 +299,7 @@ class XLNetConverter(SpmConverter):
]
def normalizer(self, proto):
# TODO Missing Replace quotes
normalizers = []
normalizers = [Replace("``", '"'), Replace("''", '"')]
if not self.original_tokenizer.keep_accents:
normalizers.append(NFKD())
normalizers.append(StripAccents())
@ -384,9 +383,6 @@ def check(pretrained, filename):
for i, line in enumerate(f):
line = line.strip()
# TODO in normalizer
line = line.replace("``", '"').replace("''", '"')
start = now()
ids = transformer_tokenizer.encode(line)
trans = now()