mirror of
https://github.com/mii443/tokenizers.git
synced 2025-08-22 16:25:30 +00:00
Python - PyStrip can get/set its attributes
This commit is contained in:
@ -359,6 +359,26 @@ impl PyLowercase {
|
||||
pub struct PyStrip {}
|
||||
#[pymethods]
|
||||
impl PyStrip {
|
||||
#[getter]
|
||||
fn get_left(self_: PyRef<Self>) -> bool {
|
||||
getter!(self_, StripNormalizer, strip_left)
|
||||
}
|
||||
|
||||
#[setter]
|
||||
fn set_left(self_: PyRef<Self>, left: bool) {
|
||||
setter!(self_, StripNormalizer, strip_left, left)
|
||||
}
|
||||
|
||||
#[getter]
|
||||
fn get_right(self_: PyRef<Self>) -> bool {
|
||||
getter!(self_, StripNormalizer, strip_right)
|
||||
}
|
||||
|
||||
#[setter]
|
||||
fn set_right(self_: PyRef<Self>, right: bool) {
|
||||
setter!(self_, StripNormalizer, strip_right, right)
|
||||
}
|
||||
|
||||
#[new]
|
||||
#[args(left = "true", right = "true")]
|
||||
fn new(left: bool, right: bool) -> PyResult<(Self, PyNormalizer)> {
|
||||
|
@ -115,6 +115,18 @@ class TestStrip:
|
||||
output = normalizer.normalize_str(" hello ")
|
||||
assert output == "hello"
|
||||
|
||||
def test_can_modify(self):
|
||||
normalizer = Strip(left=True, right=True)
|
||||
|
||||
assert normalizer.left == True
|
||||
assert normalizer.right == True
|
||||
|
||||
# Modify these
|
||||
normalizer.left = False
|
||||
assert normalizer.left == False
|
||||
normalizer.right = False
|
||||
assert normalizer.right == False
|
||||
|
||||
|
||||
class TestCustomNormalizer:
|
||||
class BadCustomNormalizer:
|
||||
|
@ -4,9 +4,10 @@ use unicode_normalization_alignments::char::is_combining_mark;
|
||||
|
||||
#[derive(Copy, Clone, Debug, Deserialize, Serialize)]
|
||||
#[serde(tag = "type")]
|
||||
#[non_exhaustive]
|
||||
pub struct Strip {
|
||||
strip_left: bool,
|
||||
strip_right: bool,
|
||||
pub strip_left: bool,
|
||||
pub strip_right: bool,
|
||||
}
|
||||
|
||||
impl Strip {
|
||||
|
Reference in New Issue
Block a user