mii/tokenizers
1
0
Fork 0
mirror of https://github.com/mii443/tokenizers.git synced 2025-09-02 07:19:24 +00:00
Code Issues Packages Projects Releases Wiki Activity

Python - RustFmt

Browse Source
This commit is contained in:
Anthony MOI
2020-02-18 14:31:56 -05:00
parent cdd8f49120
commit f263d7651f
6 changed files with 27 additions and 26 deletions
Download Patch File Download Diff File Expand all files Collapse all files

26
bindings/python/src/tokenizer.rs
View File

@ -39,10 +39,10 @@ impl Tokenizer {
}
fn num_special_tokens_to_add(&self, is_pair: bool) -> PyResult<usize> {
Ok(self.tokenizer
.get_post_processor()
.map_or(0, |p| p.as_ref().added_tokens(is_pair))
)
Ok(self
.tokenizer
.get_post_processor()
.map_or(0, |p| p.as_ref().added_tokens(is_pair)))
}
#[args(kwargs = "**")]
@ -197,10 +197,11 @@ impl Tokenizer {
}
fn decode(&self, ids: Vec<u32>, skip_special_tokens: Option<bool>) -> PyResult<String> {
ToPyResult(self.tokenizer.decode(
ids,
skip_special_tokens.unwrap_or(true),
)).into()
ToPyResult(
self.tokenizer
.decode(ids, skip_special_tokens.unwrap_or(true)),
)
.into()
}
fn decode_batch(
@ -208,10 +209,11 @@ impl Tokenizer {
sentences: Vec<Vec<u32>>,
skip_special_tokens: Option<bool>,
) -> PyResult<Vec<String>> {
ToPyResult(self.tokenizer.decode_batch(
sentences,
skip_special_tokens.unwrap_or(true),
)).into()
ToPyResult(
self.tokenizer
.decode_batch(sentences, skip_special_tokens.unwrap_or(true)),
)
.into()
}
fn token_to_id(&self, token: &str) -> Option<u32> {