mii/tokenizers
1
0
Fork 0
mirror of https://github.com/mii443/tokenizers.git synced 2025-08-22 16:25:30 +00:00
Code Issues Packages Projects Releases Wiki Activity

Handle when precompiled charsmap is empty (#1308)

Browse Source 
* Handle when precompiled charsmap is empty

* Black

---------

Co-authored-by: Nicolas Patry <patry.nicolas@protonmail.com>
This commit is contained in:
Kelly Marchisio
2023-07-31 13:35:24 +01:00
committed by GitHub
parent c2664ae13f
commit efea6c7246
1 changed files with 9 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
bindings/python/py_src/tokenizers/implementations/sentencepiece_unigram.py
View File

@ -173,12 +173,15 @@ class SentencePieceUnigramTokenizer(BaseTokenizer):
tokenizer = Tokenizer(Unigram(vocab, unk_id, byte_fallback)) tokenizer = Tokenizer(Unigram(vocab, unk_id, byte_fallback))
if precompiled_charsmap:
tokenizer.normalizer = normalizers.Sequence( tokenizer.normalizer = normalizers.Sequence(
[ [
normalizers.Precompiled(precompiled_charsmap), normalizers.Precompiled(precompiled_charsmap),
normalizers.Replace(Regex(" {2,}"), " "), normalizers.Replace(Regex(" {2,}"), " "),
] ]
) )
else:
tokenizer.normalizer = normalizers.Sequence([normalizers.Replace(Regex(" {2,}"), " ")])
tokenizer.pre_tokenizer = pre_tokenizers.Metaspace(replacement=replacement, add_prefix_space=add_prefix_space) tokenizer.pre_tokenizer = pre_tokenizers.Metaspace(replacement=replacement, add_prefix_space=add_prefix_space)
tokenizer.decoder = decoders.Metaspace(replacement=replacement, add_prefix_space=add_prefix_space) tokenizer.decoder = decoders.Metaspace(replacement=replacement, add_prefix_space=add_prefix_space)