mii/tokenizers
1
0
Fork 0
mirror of https://github.com/mii443/tokenizers.git synced 2025-12-05 12:18:20 +00:00
Code Issues Packages Projects Releases Wiki Activity

Addressing @n1t0's comments.

Browse Source
This commit is contained in:
Nicolas Patry
2020-09-04 11:57:01 +02:00
parent 7b2caca764
commit efa20202dc
3 changed files with 6 additions and 19 deletions
Download Patch File Download Diff File Expand all files Collapse all files

17
bindings/python/src/pre_tokenizers.rs
View File

@@ -288,21 +288,8 @@ pub struct PyDigits {}
#[pymethods]
impl PyDigits {
#[new]
#[args(kwargs = "**")]
fn new(kwargs: Option<&PyDict>) -> PyResult<(Self, PyPreTokenizer)> {
let mut individual_digits = false;
if let Some(kwargs) = kwargs {
for (key, value) in kwargs {
let key: &str = key.extract()?;
match key {
"individual_digits" => {
individual_digits = value.extract()?;
}
_ => println!("Ignored unknown kwarg option {}", key),
}
}
}
#[args(individual_digits = false)]
fn new(individual_digits: bool) -> PyResult<(Self, PyPreTokenizer)> {
Ok((PyDigits {}, Digits::new(individual_digits).into()))
}
}

4
bindings/python/tests/bindings/test_pre_tokenizers.py
View File

@@ -116,6 +116,6 @@ class TestDigits:
assert Digits() is not None
assert isinstance(Digits(), PreTokenizer)
assert isinstance(Digits(), Digits)
assert isinstance(Digits(individual_digits=True), Digits)
assert isinstance(Digits(individual_digits=False), Digits)
assert isinstance(Digits(True), Digits)
assert isinstance(Digits(False), Digits)
assert isinstance(pickle.loads(pickle.dumps(Digits())), Digits)