mii/tokenizers
1
0
Fork 0
mirror of https://github.com/mii443/tokenizers.git synced 2025-08-22 16:25:30 +00:00
Code Issues Packages Projects Releases Wiki Activity

Fix strip python type (#1602)

Browse Source 
* update

* the fix

* Revert "update"

This reverts commit 4c2f32f116479b0ec8ccd7c832f86cbc8787d8a9.

* add a test and rebase

* style

* oups
This commit is contained in:
Arthur
2024-08-07 15:36:28 +02:00
committed by GitHub
parent bded212356
commit 49dafd707e
2 changed files with 4 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
bindings/python/src/normalizers.rs
View File

@ -65,7 +65,7 @@ impl PyNormalizer {
Py::new(py, (PyBertNormalizer {}, base))?.into_py(py)
}
NormalizerWrapper::StripNormalizer(_) => {
Py::new(py, (PyBertNormalizer {}, base))?.into_py(py)
Py::new(py, (PyStrip {}, base))?.into_py(py)
}
NormalizerWrapper::Prepend(_) => Py::new(py, (PyPrepend {}, base))?.into_py(py),
NormalizerWrapper::ByteLevel(_) => {

5
bindings/python/tests/bindings/test_normalizers.py
View File

@ -68,12 +68,13 @@ class TestSequence:
assert output == "hello"
def test_items(self):
normalizers = Sequence([BertNormalizer(True, True), Prepend()])
normalizers = Sequence([BertNormalizer(True, True), Prepend(), Strip()])
assert normalizers[1].__class__ == Prepend
normalizers[0].lowercase = False
assert not normalizers[0].lowercase
assert normalizers[2].__class__ == Strip
with pytest.raises(IndexError):
print(normalizers[2])
print(normalizers[3])
class TestLowercase: