Fix SentencePiece tokenizers conversion

2025-12-16 17:18:43 +00:00 · 2021-02-03 09:57:41 -05:00
parent fc0a50a272
commit 96b9972842
4 changed files with 33 additions and 45 deletions
--- a/bindings/python/CHANGELOG.md
+++ b/bindings/python/CHANGELOG.md
@@ -4,6 +4,11 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).

+## [Unreleased]
+
+### Fixed
+- [#616]: Fix SentencePiece tokenizers conversion
+
 ## [0.10.0]

 ### Added
@@ -22,7 +27,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 `tokenizer.model.dropout = 0.1`)
 - [#538]: The API Reference has been improved and is now up-to-date.

-## Fixed
+### Fixed
 - [#519]: During training, the `Model` is now trained in-place. This fixes several bugs that were
 forcing to reload the `Model` after a training.
 - [#539]: Fix `BaseTokenizer` enable_truncation docstring
@@ -293,6 +298,7 @@ delimiter (Works like `.split(delimiter)`)
 - Fix a bug that was causing crashes in Python 3.5


+[#616]: https://github.com/huggingface/tokenizers/pull/616
 [#590]: https://github.com/huggingface/tokenizers/pull/590
 [#574]: https://github.com/huggingface/tokenizers/pull/574
 [#544]: https://github.com/huggingface/tokenizers/pull/544
--- a/bindings/python/py_src/tokenizers/implementations/sentencepiece_unigram.py
+++ b/bindings/python/py_src/tokenizers/implementations/sentencepiece_unigram.py
@@ -1,11 +1,4 @@
-from tokenizers import (
-    Tokenizer,
-    AddedToken,
-    pre_tokenizers,
-    decoders,
-    trainers,
-    normalizers,
-)
+from tokenizers import Tokenizer, AddedToken, pre_tokenizers, decoders, trainers, normalizers, Regex
 import os
 from tokenizers.models import Unigram
 import json
@@ -33,18 +26,10 @@ class SentencePieceUnigramTokenizer(BaseTokenizer):
            tokenizer = Tokenizer(Unigram())

        tokenizer.normalizer = normalizers.Sequence(
-            [
-                normalizers.Nmt(),
-                normalizers.NFKC(),
-            ]
+            [normalizers.Nmt(), normalizers.NFKC(), normalizers.Replace(Regex(" {2,}"), " ")]
        )
-        tokenizer.pre_tokenizer = pre_tokenizers.Sequence(
-            [
-                pre_tokenizers.WhitespaceSplit(),
-                pre_tokenizers.Metaspace(
-                    replacement=replacement, add_prefix_space=add_prefix_space
-                ),
-            ]
+        tokenizer.pre_tokenizer = pre_tokenizers.Metaspace(
+            replacement=replacement, add_prefix_space=add_prefix_space
        )
        tokenizer.decoder = decoders.Metaspace(
            replacement=replacement, add_prefix_space=add_prefix_space
@@ -124,15 +109,15 @@ class SentencePieceUnigramTokenizer(BaseTokenizer):

        tokenizer = Tokenizer(Unigram(vocab, unk_id))

-        tokenizer.normalizer = normalizers.Precompiled(precompiled_charsmap)
-        tokenizer.pre_tokenizer = pre_tokenizers.Sequence(
+        tokenizer.normalizer = normalizers.Sequence(
            [
-                pre_tokenizers.WhitespaceSplit(),
-                pre_tokenizers.Metaspace(
-                    replacement=replacement, add_prefix_space=add_prefix_space
-                ),
+                normalizers.Precompiled(precompiled_charsmap),
+                normalizers.Replace(Regex(" {2,}"), " "),
            ]
        )
+        tokenizer.pre_tokenizer = pre_tokenizers.Metaspace(
+            replacement=replacement, add_prefix_space=add_prefix_space
+        )
        tokenizer.decoder = decoders.Metaspace(
            replacement=replacement, add_prefix_space=add_prefix_space
        )
--- a/bindings/python/scripts/convert.py
+++ b/bindings/python/scripts/convert.py
@@ -3,7 +3,7 @@ from tokenizers.implementations import SentencePieceUnigramTokenizer, BaseTokeni
 from tokenizers.processors import TemplateProcessing
 from tokenizers.models import Unigram, BPE
 from tokenizers import decoders
-from tokenizers import Tokenizer
+from tokenizers import Tokenizer, Regex
 from tokenizers.normalizers import (
    StripAccents,
    NFKD,
@@ -81,7 +81,7 @@ class SpmConverter(Converter):
        elif model_type == 2:
            vocab, merges = SentencePieceExtractor(self.original_tokenizer.vocab_file).extract()
            tokenizer = Tokenizer(
-                BPE(vocab, merges, unk_token=proto.trainer_spec.unk_piece, fuse_unk=True,)
+                BPE(vocab, merges, unk_token=proto.trainer_spec.unk_piece, fuse_unk=True)
            )
        else:
            raise Exception(
@@ -92,7 +92,7 @@ class SpmConverter(Converter):

    def normalizer(self, proto):
        precompiled_charsmap = proto.normalizer_spec.precompiled_charsmap
-        return Precompiled(precompiled_charsmap)
+        return Sequence([Precompiled(precompiled_charsmap), Replace(Regex(" {2,}"), " ")])

    def post_processor(self, tokenizer):
        return None
@@ -105,11 +105,8 @@ class SpmConverter(Converter):

        replacement = "▁"
        add_prefix_space = True
-        tokenizer.pre_tokenizer = PSequence(
-            [
-                WhitespaceSplit(),
-                Metaspace(replacement=replacement, add_prefix_space=add_prefix_space),
-            ]
+        tokenizer.pre_tokenizer = Metaspace(
+            replacement=replacement, add_prefix_space=add_prefix_space
        )
        tokenizer.decoder = decoders.Metaspace(
            replacement=replacement, add_prefix_space=add_prefix_space
@@ -134,7 +131,7 @@ class AlbertConverter(SpmConverter):
        ]

    def normalizer(self, proto):
-        normalizers = [Replace("``", '"'), Replace("''", '"')]
+        normalizers = [Replace("``", '"'), Replace("''", '"'), Replace(Regex(" {2,}"), " ")]
        if not self.original_tokenizer.keep_accents:
            normalizers.append(NFKD())
            normalizers.append(StripAccents())
@@ -270,7 +267,7 @@ class XLNetConverter(SpmConverter):
        ]

    def normalizer(self, proto):
-        normalizers = [Replace("``", '"'), Replace("''", '"')]
+        normalizers = [Replace("``", '"'), Replace("''", '"'), Replace(Regex(" {2,}"), " ")]
        if not self.original_tokenizer.keep_accents:
            normalizers.append(NFKD())
            normalizers.append(StripAccents())
@@ -316,7 +313,7 @@ class PegasusConverter(SpmConverter):
        return TemplateProcessing(
            seq_a=["$0", eos],
            seq_b=["$1", eos],
-            special_tokens=[(eos, tokenizer.get_vocab()[eos]),],
+            special_tokens=[(eos, tokenizer.get_vocab()[eos])],
        )


@@ -325,7 +322,7 @@ class T5Converter(SpmConverter):
        return TemplateProcessing(
            seq_a=["$0", "</s>"],
            seq_b=["$1", "</s>"],
-            special_tokens=[("</s>", tokenizer.get_vocab()["</s>"]),],
+            special_tokens=[("</s>", tokenizer.get_vocab()["</s>"])],
        )


--- a/bindings/python/scripts/spm_parity_check.py
+++ b/bindings/python/scripts/spm_parity_check.py
@@ -131,12 +131,12 @@ def check_diff(spm_diff, tok_diff, sp, tok):
    if spm_diff == list(reversed(tok_diff)):
        # AAA -> AA+A vs A+AA case.
        return True
-    # elif len(spm_diff) == len(tok_diff) and tok.decode(spm_diff) == tok.decode(
-    #     tok_diff
-    # ):
-    #     # Second order OK
-    #     # Barrich -> Barr + ich vs Bar + rich
-    #     return True
+    elif len(spm_diff) == len(tok_diff) and tok.decode(spm_diff) == tok.decode(
+        tok_diff
+    ):
+        # Second order OK
+        # Barrich -> Barr + ich vs Bar + rich
+        return True
    spm_reencoded = sp.encode(sp.decode(spm_diff))
    tok_reencoded = tok.encode(tok.decode(spm_diff)).ids
    if spm_reencoded != spm_diff and spm_reencoded == tok_reencoded:
@@ -265,7 +265,7 @@ def check_encode(args):
            else:
                perfect += 1

-            assert ids == encoded.ids, f"line {i}: {line} : {ids} != {encoded.ids}"
+            assert ids == encoded.ids, f"line {i}: {line} : \n\n{ids}\n{encoded.ids}\n{list(zip(encoded.ids, encoded.tokens))}"

    print(f"({perfect} / {imperfect} / {wrong} ----- {perfect + imperfect + wrong})")
    total = perfect + imperfect + wrong