mirror of
https://github.com/mii443/tokenizers.git
synced 2025-08-23 00:35:35 +00:00
Python - PyStrip can get/set its attributes
This commit is contained in:
@ -359,6 +359,26 @@ impl PyLowercase {
|
|||||||
pub struct PyStrip {}
|
pub struct PyStrip {}
|
||||||
#[pymethods]
|
#[pymethods]
|
||||||
impl PyStrip {
|
impl PyStrip {
|
||||||
|
#[getter]
|
||||||
|
fn get_left(self_: PyRef<Self>) -> bool {
|
||||||
|
getter!(self_, StripNormalizer, strip_left)
|
||||||
|
}
|
||||||
|
|
||||||
|
#[setter]
|
||||||
|
fn set_left(self_: PyRef<Self>, left: bool) {
|
||||||
|
setter!(self_, StripNormalizer, strip_left, left)
|
||||||
|
}
|
||||||
|
|
||||||
|
#[getter]
|
||||||
|
fn get_right(self_: PyRef<Self>) -> bool {
|
||||||
|
getter!(self_, StripNormalizer, strip_right)
|
||||||
|
}
|
||||||
|
|
||||||
|
#[setter]
|
||||||
|
fn set_right(self_: PyRef<Self>, right: bool) {
|
||||||
|
setter!(self_, StripNormalizer, strip_right, right)
|
||||||
|
}
|
||||||
|
|
||||||
#[new]
|
#[new]
|
||||||
#[args(left = "true", right = "true")]
|
#[args(left = "true", right = "true")]
|
||||||
fn new(left: bool, right: bool) -> PyResult<(Self, PyNormalizer)> {
|
fn new(left: bool, right: bool) -> PyResult<(Self, PyNormalizer)> {
|
||||||
|
@ -115,6 +115,18 @@ class TestStrip:
|
|||||||
output = normalizer.normalize_str(" hello ")
|
output = normalizer.normalize_str(" hello ")
|
||||||
assert output == "hello"
|
assert output == "hello"
|
||||||
|
|
||||||
|
def test_can_modify(self):
|
||||||
|
normalizer = Strip(left=True, right=True)
|
||||||
|
|
||||||
|
assert normalizer.left == True
|
||||||
|
assert normalizer.right == True
|
||||||
|
|
||||||
|
# Modify these
|
||||||
|
normalizer.left = False
|
||||||
|
assert normalizer.left == False
|
||||||
|
normalizer.right = False
|
||||||
|
assert normalizer.right == False
|
||||||
|
|
||||||
|
|
||||||
class TestCustomNormalizer:
|
class TestCustomNormalizer:
|
||||||
class BadCustomNormalizer:
|
class BadCustomNormalizer:
|
||||||
|
@ -4,9 +4,10 @@ use unicode_normalization_alignments::char::is_combining_mark;
|
|||||||
|
|
||||||
#[derive(Copy, Clone, Debug, Deserialize, Serialize)]
|
#[derive(Copy, Clone, Debug, Deserialize, Serialize)]
|
||||||
#[serde(tag = "type")]
|
#[serde(tag = "type")]
|
||||||
|
#[non_exhaustive]
|
||||||
pub struct Strip {
|
pub struct Strip {
|
||||||
strip_left: bool,
|
pub strip_left: bool,
|
||||||
strip_right: bool,
|
pub strip_right: bool,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl Strip {
|
impl Strip {
|
||||||
|
Reference in New Issue
Block a user