mii/tokenizers
1
0
Fork 0
mirror of https://github.com/mii443/tokenizers.git synced 2025-08-22 16:25:30 +00:00
Code Issues Packages Projects Releases Wiki Activity

Fix strip python type (#1602)

Browse Source 
* update

* the fix

* Revert "update"

This reverts commit 4c2f32f116479b0ec8ccd7c832f86cbc8787d8a9.

* add a test and rebase

* style

* oups
This commit is contained in:
Arthur
2024-08-07 15:36:28 +02:00
committed by GitHub
parent bded212356
commit 49dafd707e
2 changed files with 4 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
bindings/python/src/normalizers.rs
View File

@ -65,7 +65,7 @@ impl PyNormalizer {
Py::new(py, (PyBertNormalizer {}, base))?.into_py(py) Py::new(py, (PyBertNormalizer {}, base))?.into_py(py)
} }
NormalizerWrapper::StripNormalizer(_) => { NormalizerWrapper::StripNormalizer(_) => {
Py::new(py, (PyBertNormalizer {}, base))?.into_py(py) Py::new(py, (PyStrip {}, base))?.into_py(py)
} }
NormalizerWrapper::Prepend(_) => Py::new(py, (PyPrepend {}, base))?.into_py(py), NormalizerWrapper::Prepend(_) => Py::new(py, (PyPrepend {}, base))?.into_py(py),
NormalizerWrapper::ByteLevel(_) => { NormalizerWrapper::ByteLevel(_) => {

5
bindings/python/tests/bindings/test_normalizers.py
View File

@ -68,12 +68,13 @@ class TestSequence:
assert output == "hello" assert output == "hello"
def test_items(self): def test_items(self):
normalizers = Sequence([BertNormalizer(True, True), Prepend()]) normalizers = Sequence([BertNormalizer(True, True), Prepend(), Strip()])
assert normalizers[1].__class__ == Prepend assert normalizers[1].__class__ == Prepend
normalizers[0].lowercase = False normalizers[0].lowercase = False
assert not normalizers[0].lowercase assert not normalizers[0].lowercase
assert normalizers[2].__class__ == Strip
with pytest.raises(IndexError): with pytest.raises(IndexError):
print(normalizers[2]) print(normalizers[3])
class TestLowercase: class TestLowercase: