mii/tokenizers
1
0
Fork 0
mirror of https://github.com/mii443/tokenizers.git synced 2025-12-06 04:38:23 +00:00
Code Issues Packages Projects Releases Wiki Activity

Python - Improve decode/decode_batch API

Browse Source
This commit is contained in:
Anthony MOI
2020-01-06 16:39:36 -05:00
parent 1a083a6e6f
commit b7d0acc562
1 changed files with 20 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

22
bindings/python/src/tokenizer.rs
View File

@@ -244,15 +244,33 @@ impl Tokenizer {
.into()
}
fn decode(&self, ids: Vec<u32>, skip_special_tokens: bool) -> PyResult<String> {
#[args(kwargs = "**")]
fn decode(&self, ids: Vec<u32>, kwargs: Option<&PyDict>) -> PyResult<String> {
let mut skip_special_tokens = true;
if let Some(kwargs) = kwargs {
if let Some(skip) = kwargs.get_item("skip_special_tokens") {
skip_special_tokens = skip.extract()?;
}
}
ToPyResult(self.tokenizer.decode(ids, skip_special_tokens)).into()
}
#[args(kwargs = "**")]
fn decode_batch(
&self,
sentences: Vec<Vec<u32>>,
skip_special_tokens: bool,
kwargs: Option<&PyDict>,
) -> PyResult<Vec<String>> {
let mut skip_special_tokens = true;
if let Some(kwargs) = kwargs {
if let Some(skip) = kwargs.get_item("skip_special_tokens") {
skip_special_tokens = skip.extract()?;
}
}
ToPyResult(self.tokenizer.decode_batch(sentences, skip_special_tokens)).into()
}