mii/tokenizers
1
0
Fork 0
mirror of https://github.com/mii443/tokenizers.git synced 2025-08-22 16:25:30 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix build

Browse Source
This commit is contained in:
Arthur Zucker
2023-09-01 21:22:07 +00:00
parent 9f0c703f03
commit 39bd27e673
2 changed files with 12 additions and 12 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
bindings/python/src/tokenizer.rs
View File

@ -119,7 +119,7 @@ impl From<tk::AddedToken> for PyAddedToken {
lstrip: Some(token.lstrip),
rstrip: Some(token.rstrip),
normalized: Some(token.normalized),
special: Some(token.special),
special: token.special,
}
}
}
@ -127,7 +127,7 @@ impl From<tk::AddedToken> for PyAddedToken {
#[pymethods]
impl PyAddedToken {
#[new]
#[pyo3(signature = (content=None, **kwargs), text_signature = "(self, content, single_word=False, lstrip=False, rstrip=False, normalized=True)")]
#[pyo3(signature = (content=None, **kwargs), text_signature = "(self, content, single_word=False, lstrip=False, rstrip=False, normalized=True, special=False)")]
fn __new__(content: Option<&str>, kwargs: Option<&PyDict>) -> PyResult<Self> {
let mut token = PyAddedToken::from(content.unwrap_or(""), None);
@ -139,7 +139,7 @@ impl PyAddedToken {
"lstrip" => token.lstrip = Some(value.extract()?),
"rstrip" => token.rstrip = Some(value.extract()?),
"normalized" => token.normalized = Some(value.extract()?),
"special" => token.special = Some(value.extract()?),
"special" => token.special = value.extract()?,
_ => println!("Ignored unknown kwarg option {}", key),
}
}
@ -163,7 +163,7 @@ impl PyAddedToken {
"lstrip" => self.lstrip = Some(value.extract()?),
"rstrip" => self.rstrip = Some(value.extract()?),
"normalized" => self.normalized = Some(value.extract()?),
"special" => self.special = Some(value.extract()?),
"special" => self.special = value.extract()?,
_ => {}
}
}

16
bindings/python/src/trainers.rs
View File

@ -226,7 +226,7 @@ impl PyBpeTrainer {
if let Ok(content) = token.extract::<String>() {
Ok(tk::tokenizer::AddedToken::from(content, true))
} else if let Ok(mut token) = token.extract::<PyRefMut<PyAddedToken>>() {
token.is_special_token = true;
token.special = true;
Ok(token.get_token())
} else {
Err(exceptions::PyTypeError::new_err(
@ -319,7 +319,7 @@ impl PyBpeTrainer {
} else if let Ok(mut token) =
token.extract::<PyRefMut<PyAddedToken>>()
{
token.is_special_token = true;
token.special = true;
Ok(token.get_token())
} else {
Err(exceptions::PyTypeError::new_err(
@ -440,7 +440,7 @@ impl PyWordPieceTrainer {
if let Ok(content) = token.extract::<String>() {
Ok(tk::tokenizer::AddedToken::from(content, true))
} else if let Ok(mut token) = token.extract::<PyRefMut<PyAddedToken>>() {
token.is_special_token = true;
token.special = true;
Ok(token.get_token())
} else {
Err(exceptions::PyTypeError::new_err(
@ -526,7 +526,7 @@ impl PyWordPieceTrainer {
} else if let Ok(mut token) =
token.extract::<PyRefMut<PyAddedToken>>()
{
token.is_special_token = true;
token.special = true;
Ok(token.get_token())
} else {
Err(exceptions::PyTypeError::new_err(
@ -632,7 +632,7 @@ impl PyWordLevelTrainer {
if let Ok(content) = token.extract::<String>() {
Ok(tk::tokenizer::AddedToken::from(content, true))
} else if let Ok(mut token) = token.extract::<PyRefMut<PyAddedToken>>() {
token.is_special_token = true;
token.special = true;
Ok(token.get_token())
} else {
Err(exceptions::PyTypeError::new_err(
@ -673,7 +673,7 @@ impl PyWordLevelTrainer {
} else if let Ok(mut token) =
token.extract::<PyRefMut<PyAddedToken>>()
{
token.is_special_token = true;
token.special = true;
Ok(token.get_token())
} else {
Err(exceptions::PyTypeError::new_err(
@ -778,7 +778,7 @@ impl PyUnigramTrainer {
if let Ok(content) = token.extract::<String>() {
Ok(tk::tokenizer::AddedToken::from(content, true))
} else if let Ok(mut token) = token.extract::<PyRefMut<PyAddedToken>>() {
token.is_special_token = true;
token.special = true;
Ok(token.get_token())
} else {
Err(exceptions::PyTypeError::new_err(
@ -846,7 +846,7 @@ impl PyUnigramTrainer {
} else if let Ok(mut token) =
token.extract::<PyRefMut<PyAddedToken>>()
{
token.is_special_token = true;
token.special = true;
Ok(token.get_token())
} else {
Err(exceptions::PyTypeError::new_err(