mii/tokenizers
1
0
Fork 0
mirror of https://github.com/mii443/tokenizers.git synced 2025-08-22 16:25:30 +00:00
Code Issues Packages Projects Releases Wiki Activity

Fixing doc. (#1499)

Browse Source 
* Fixing doc.

* SentencePieceUnigram  and Convert.py still used sentencepiece

* stub

---------

Co-authored-by: Arthur Zucker <arthur.zucker@gmail.com>
This commit is contained in:
Nicolas Patry
2024-04-17 09:32:40 +02:00
committed by GitHub
parent 949d9e3e0e
commit 91393ef75e
7 changed files with 20 additions and 11 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
bindings/python/py_src/tokenizers/decoders/__init__.pyi
View File

@ -152,9 +152,11 @@ class Metaspace(Decoder):
The replacement character. Must be exactly one character. By default we
use the `▁` (U+2581) meta symbol (Same as in SentencePiece).
add_prefix_space (:obj:`bool`, `optional`, defaults to :obj:`True`):
prepend_scheme (:obj:`str`, `optional`, defaults to :obj:`"always"`):
Whether to add a space to the first word if there isn't already one. This
lets us treat `hello` exactly like `say hello`.
Choices: "always", "never", "first". First means the space is only added on the first
token (relevant when special tokens are used or other pre_tokenizer are used).
"""
def __init__(self, replacement="▁", prepend_scheme="always", split=True):
pass

5
bindings/python/py_src/tokenizers/implementations/sentencepiece_unigram.py
View File

@ -183,8 +183,9 @@ class SentencePieceUnigramTokenizer(BaseTokenizer):
)
else:
tokenizer.normalizer = normalizers.Sequence([normalizers.Replace(Regex(" {2,}"), " ")])
tokenizer.pre_tokenizer = pre_tokenizers.Metaspace(replacement=replacement, add_prefix_space=add_prefix_space)
tokenizer.decoder = decoders.Metaspace(replacement=replacement, add_prefix_space=add_prefix_space)
prepend_scheme = "always" if add_prefix_space else "never"
tokenizer.pre_tokenizer = pre_tokenizers.Metaspace(replacement=replacement, prepend_scheme=prepend_scheme)
tokenizer.decoder = decoders.Metaspace(replacement=replacement, prepend_scheme=prepend_scheme)
parameters = {
"model": "SentencePieceUnigram",

5
bindings/python/py_src/tokenizers/pre_tokenizers/__init__.pyi
View File

@ -270,9 +270,12 @@ class Metaspace(PreTokenizer):
The replacement character. Must be exactly one character. By default we
use the `▁` (U+2581) meta symbol (Same as in SentencePiece).
add_prefix_space (:obj:`bool`, `optional`, defaults to :obj:`True`):
prepend_scheme (:obj:`str`, `optional`, defaults to :obj:`"always"`):
Whether to add a space to the first word if there isn't already one. This
lets us treat `hello` exactly like `say hello`.
Choices: "always", "never", "first". First means the space is only added on the first
token (relevant when special tokens are used or other pre_tokenizer are used).
"""
def __init__(self, replacement="_", prepend_scheme="always", split=True):
pass

6
bindings/python/scripts/convert.py
View File

@ -102,9 +102,9 @@ class SpmConverter(Converter):
tokenizer.normalizer = self.normalizer(self.proto)
replacement = "▁"
add_prefix_space = True
tokenizer.pre_tokenizer = Metaspace(replacement=replacement, add_prefix_space=add_prefix_space)
tokenizer.decoder = decoders.Metaspace(replacement=replacement, add_prefix_space=add_prefix_space)
prepend_scheme = "always"
tokenizer.pre_tokenizer = Metaspace(replacement=replacement, prepend_scheme=prepend_scheme)
tokenizer.decoder = decoders.Metaspace(replacement=replacement, prepend_scheme=prepend_scheme)
post_processor = self.post_processor(tokenizer)
if post_processor:
tokenizer.post_processor = post_processor

4
bindings/python/src/decoders.rs
View File

@ -304,9 +304,11 @@ impl PyStrip {
/// The replacement character. Must be exactly one character. By default we
/// use the `▁` (U+2581) meta symbol (Same as in SentencePiece).
///
/// add_prefix_space (:obj:`bool`, `optional`, defaults to :obj:`True`):
/// prepend_scheme (:obj:`str`, `optional`, defaults to :obj:`"always"`):
/// Whether to add a space to the first word if there isn't already one. This
/// lets us treat `hello` exactly like `say hello`.
/// Choices: "always", "never", "first". First means the space is only added on the first
/// token (relevant when special tokens are used or other pre_tokenizer are used).
#[pyclass(extends=PyDecoder, module = "tokenizers.decoders", name = "Metaspace")]
pub struct PyMetaspaceDec {}
#[pymethods]

5
bindings/python/src/pre_tokenizers.rs
View File

@ -477,9 +477,12 @@ pub(crate) fn from_string(string: String) -> Result<PrependScheme, PyErr> {
/// The replacement character. Must be exactly one character. By default we
/// use the `▁` (U+2581) meta symbol (Same as in SentencePiece).
///
/// add_prefix_space (:obj:`bool`, `optional`, defaults to :obj:`True`):
/// prepend_scheme (:obj:`str`, `optional`, defaults to :obj:`"always"`):
/// Whether to add a space to the first word if there isn't already one. This
/// lets us treat `hello` exactly like `say hello`.
/// Choices: "always", "never", "first". First means the space is only added on the first
/// token (relevant when special tokens are used or other pre_tokenizer are used).
///
#[pyclass(extends=PyPreTokenizer, module = "tokenizers.pre_tokenizers", name = "Metaspace")]
pub struct PyMetaspace {}
#[pymethods]

2
bindings/python/tests/bindings/test_pre_tokenizers.py
View File

@ -109,8 +109,6 @@ class TestMetaspace:
# Modify these
pretok.replacement = "%"
assert pretok.replacement == "%"
pretok.add_prefix_space = True
assert pretok.add_prefix_space == True
pretok.prepend_scheme = "first"
assert pretok.prepend_scheme == "first"
pretok.split = True