mii/tokenizers
1
0
Fork 0
mirror of https://github.com/mii443/tokenizers.git synced 2025-08-22 16:25:30 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #70 from huggingface/python-decode-kwargs-fix

Browse Source 
Remove kwargs mapping on Tokenizer decode/decode_batch
This commit is contained in:
MOI Anthony
2020-01-15 09:40:42 -05:00
committed by GitHub
parent a779714a9e 4839154145
commit 65b35385f8
1 changed files with 10 additions and 22 deletions
Download Patch File Download Diff File Expand all files Collapse all files

32
bindings/python/src/tokenizer.rs
View File

@ -189,34 +189,22 @@ impl Tokenizer {
.into()
}
#[args(kwargs = "**")]
fn decode(&self, ids: Vec<u32>, kwargs: Option<&PyDict>) -> PyResult<String> {
let mut skip_special_tokens = true;
if let Some(kwargs) = kwargs {
if let Some(skip) = kwargs.get_item("skip_special_tokens") {
skip_special_tokens = skip.extract()?;
}
}
ToPyResult(self.tokenizer.decode(ids, skip_special_tokens)).into()
fn decode(&self, ids: Vec<u32>, skip_special_tokens: Option<bool>) -> PyResult<String> {
ToPyResult(self.tokenizer.decode(
ids,
skip_special_tokens.unwrap_or(true),
)).into()
}
#[args(kwargs = "**")]
fn decode_batch(
&self,
sentences: Vec<Vec<u32>>,
kwargs: Option<&PyDict>,
skip_special_tokens: Option<bool>,
) -> PyResult<Vec<String>> {
let mut skip_special_tokens = true;
if let Some(kwargs) = kwargs {
if let Some(skip) = kwargs.get_item("skip_special_tokens") {
skip_special_tokens = skip.extract()?;
}
}
ToPyResult(self.tokenizer.decode_batch(sentences, skip_special_tokens)).into()
ToPyResult(self.tokenizer.decode_batch(
sentences,
skip_special_tokens.unwrap_or(true),
)).into()
}
fn token_to_id(&self, token: &str) -> Option<u32> {