mii/tokenizers
1
0
Fork 0
mirror of https://github.com/mii443/tokenizers.git synced 2025-08-22 16:25:30 +00:00
Code Issues Packages Projects Releases Wiki Activity

Preventing yelling on empty OrderedVocab (triggered by pickle.dumps). (#963)

Browse Source
This commit is contained in:
Nicolas Patry
2022-03-24 17:09:18 +01:00
committed by GitHub
parent a5f644616b
commit 28fe0e40e7
1 changed files with 6 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
tokenizers/src/models/mod.rs
View File

@ -34,8 +34,8 @@ impl<'a> Serialize for OrderedVocabIter<'a> {
S: Serializer,
{
// There could be holes so max + 1 is more correct than vocab_r.len()
let max = self.vocab_r.iter().map(|(key, _)| key).max().unwrap_or(&0) + 1;
let iter = (0..max).filter_map(|i| {
if let Some(max) = self.vocab_r.iter().map(|(key, _)| key).max() {
let iter = (0..*max + 1).filter_map(|i| {
if let Some(token) = self.vocab_r.get(&i){
Some((token, i))
}else{
@ -44,7 +44,10 @@ impl<'a> Serialize for OrderedVocabIter<'a> {
None
}
});
serializer.collect_map(iter)
serializer.collect_map(iter)
} else {
serializer.collect_map(std::iter::empty::<(&str, u32)>())
}
}
}