mii/tokenizers
1
0
Fork 0
mirror of https://github.com/mii443/tokenizers.git synced 2025-12-06 04:38:23 +00:00
Code Issues Packages Projects Releases Wiki Activity

Fixing convert/check scripts.

Browse Source
This commit is contained in:
Nicolas Patry
2020-09-18 11:51:46 +02:00
parent c0b9229833
commit c59b216baa
2 changed files with 6 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
bindings/python/scripts/convert.py
View File

@@ -397,7 +397,7 @@ def check(pretrained, filename):
tok_total_time += tok - trans
if ids != tok_ids:
if check_details(line, ids, tok_ids, tokenizer, transformer_tokenizer):
if check_details(line, ids, tok_ids, transformer_tokenizer, tokenizer):
continue
assert ids == tok_ids, f"Error in line {i}: {line} {ids} != {tok_ids}"