mii/tokenizers
1
0
Fork 0
mirror of https://github.com/mii443/tokenizers.git synced 2025-08-24 00:59:19 +00:00
Code Issues Packages Projects Releases Wiki Activity

Python - Fix some clippy warnings

Browse Source
This commit is contained in:
Anthony MOI
2019-12-13 18:17:51 -05:00
parent 1a604cdbee
commit ed7e3999d2
1 changed files with 8 additions and 9 deletions
Download Patch File Download Diff File Expand all files Collapse all files

17
bindings/python/src/tokenizer.rs
View File

@ -72,12 +72,12 @@ impl Tokenizer {
fn encode(&self, sentence: &str, pair: Option<&str>) -> PyResult<Encoding> {
ToPyResult(
self.tokenizer
.encode(if pair.is_some() {
tk::tokenizer::EncodeInput::Dual(sentence.to_owned(), pair.unwrap().to_owned())
.encode(if let Some(pair) = pair {
tk::tokenizer::EncodeInput::Dual(sentence.to_owned(), pair.to_owned())
} else {
tk::tokenizer::EncodeInput::Single(sentence.to_owned())
})
.map(|encoding| Encoding::new(encoding)),
.map(Encoding::new),
)
.into()
}
@ -98,12 +98,11 @@ impl Tokenizer {
})
.collect::<PyResult<Vec<_>>>()?;
ToPyResult(self.tokenizer.encode_batch(inputs).map(|encodings| {
encodings
.into_iter()
.map(|encoding| Encoding::new(encoding))
.collect()
}))
ToPyResult(
self.tokenizer
.encode_batch(inputs)
.map(|encodings| encodings.into_iter().map(Encoding::new).collect()),
)
.into()
}