mii/tokenizers
1
0
Fork 0
mirror of https://github.com/mii443/tokenizers.git synced 2025-08-23 00:35:35 +00:00
Code Issues Packages Projects Releases Wiki Activity

Update PyO3 (#426)

Browse Source
This commit is contained in:
Anthony MOI
2020-09-22 12:00:20 -04:00
committed by GitHub
parent 8e220dbdd4
commit 940f8bd8fa
13 changed files with 156 additions and 178 deletions
Download Patch File Download Diff File Expand all files Collapse all files

13
bindings/python/src/trainers.rs
View File

@ -73,7 +73,7 @@ impl PyBpeTrainer {
token.is_special_token = true;
Ok(token.get_token())
} else {
Err(exceptions::Exception::py_err(
Err(exceptions::PyTypeError::new_err(
"special_tokens must be a List[Union[str, AddedToken]]",
))
}
@ -137,7 +137,7 @@ impl PyWordPieceTrainer {
token.is_special_token = true;
Ok(token.get_token())
} else {
Err(exceptions::Exception::py_err(
Err(exceptions::PyTypeError::new_err(
"special_tokens must be a List[Union[str, AddedToken]]",
))
}
@ -205,7 +205,7 @@ impl PyUnigramTrainer {
token.is_special_token = true;
Ok(token.get_token())
} else {
Err(exceptions::Exception::py_err(
Err(exceptions::PyTypeError::new_err(
"special_tokens must be a List[Union[str, AddedToken]]",
))
}
@ -220,9 +220,10 @@ impl PyUnigramTrainer {
}
}
let trainer: tokenizers::models::unigram::UnigramTrainer = builder
.build()
.map_err(|_| exceptions::Exception::py_err("Cannot build UnigramTrainer"))?;
let trainer: tokenizers::models::unigram::UnigramTrainer =
builder.build().map_err(|e| {
exceptions::PyException::new_err(format!("Cannot build UnigramTrainer: {}", e))
})?;
Ok((PyUnigramTrainer {}, PyTrainer::new(trainer.into())))
}
}