mii/tokenizers
1
0
Fork 0
mirror of https://github.com/mii443/tokenizers.git synced 2025-12-09 22:28:29 +00:00
Code Issues Packages Projects Releases Wiki Activity

Python - Improve decode/decode_batch API

Browse Source
This commit is contained in:
Anthony MOI
2020-01-06 16:39:36 -05:00
parent 1a083a6e6f
commit b7d0acc562
1 changed files with 20 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

22
bindings/python/src/tokenizer.rs
View File

@@ -244,15 +244,33 @@ impl Tokenizer {
.into() .into()
} }
fn decode(&self, ids: Vec<u32>, skip_special_tokens: bool) -> PyResult<String> { #[args(kwargs = "**")]
fn decode(&self, ids: Vec<u32>, kwargs: Option<&PyDict>) -> PyResult<String> {
let mut skip_special_tokens = true;
if let Some(kwargs) = kwargs {
if let Some(skip) = kwargs.get_item("skip_special_tokens") {
skip_special_tokens = skip.extract()?;
}
}
ToPyResult(self.tokenizer.decode(ids, skip_special_tokens)).into() ToPyResult(self.tokenizer.decode(ids, skip_special_tokens)).into()
} }
#[args(kwargs = "**")]
fn decode_batch( fn decode_batch(
&self, &self,
sentences: Vec<Vec<u32>>, sentences: Vec<Vec<u32>>,
skip_special_tokens: bool, kwargs: Option<&PyDict>,
) -> PyResult<Vec<String>> { ) -> PyResult<Vec<String>> {
let mut skip_special_tokens = true;
if let Some(kwargs) = kwargs {
if let Some(skip) = kwargs.get_item("skip_special_tokens") {
skip_special_tokens = skip.extract()?;
}
}
ToPyResult(self.tokenizer.decode_batch(sentences, skip_special_tokens)).into() ToPyResult(self.tokenizer.decode_batch(sentences, skip_special_tokens)).into()
} }