mii/tokenizers
1
0
Fork 0
mirror of https://github.com/mii443/tokenizers.git synced 2025-08-23 00:35:35 +00:00
Code Issues Packages Projects Releases Wiki Activity

Make sure we first try to extract a string

Browse Source
This commit is contained in:
Anthony MOI
2020-11-25 16:43:52 -05:00
committed by Anthony MOI
parent ed9baeabb7
commit 999067454d
1 changed files with 6 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
bindings/python/src/tokenizer.rs
View File

@ -1102,10 +1102,13 @@ impl PyTokenizer {
// - A string
iterator.iter()?.flat_map(|seq| match seq {
Ok(s) => {
if let Ok(iter) = s.iter() {
itertools::Either::Left(iter.map(|i| i?.extract::<&str>()))
if let Ok(s) = s.downcast::<PyString>() {
itertools::Either::Right(std::iter::once(s.to_str()))
} else {
itertools::Either::Right(std::iter::once(s.extract::<&str>()))
match s.iter() {
Ok(iter) => itertools::Either::Left(iter.map(|i| i?.extract::<&str>())),
Err(e) => itertools::Either::Right(std::iter::once(Err(e))),
}
}
}
Err(e) => itertools::Either::Right(std::iter::once(Err(e))),