mii/tokenizers
1
0
Fork 0
mirror of https://github.com/mii443/tokenizers.git synced 2025-08-23 00:35:35 +00:00
Code Issues Packages Projects Releases Wiki Activity

Preventing yelling on empty OrderedVocab (triggered by pickle.dumps). (#963)

Browse Source
This commit is contained in:
Nicolas Patry
2022-03-24 17:09:18 +01:00
committed by GitHub
parent a5f644616b
commit 28fe0e40e7
1 changed files with 6 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
tokenizers/src/models/mod.rs
View File

@ -34,8 +34,8 @@ impl<'a> Serialize for OrderedVocabIter<'a> {
S: Serializer, S: Serializer,
{ {
// There could be holes so max + 1 is more correct than vocab_r.len() // There could be holes so max + 1 is more correct than vocab_r.len()
let max = self.vocab_r.iter().map(|(key, _)| key).max().unwrap_or(&0) + 1; if let Some(max) = self.vocab_r.iter().map(|(key, _)| key).max() {
let iter = (0..max).filter_map(|i| { let iter = (0..*max + 1).filter_map(|i| {
if let Some(token) = self.vocab_r.get(&i){ if let Some(token) = self.vocab_r.get(&i){
Some((token, i)) Some((token, i))
}else{ }else{
@ -45,6 +45,9 @@ impl<'a> Serialize for OrderedVocabIter<'a> {
} }
}); });
serializer.collect_map(iter) serializer.collect_map(iter)
} else {
serializer.collect_map(std::iter::empty::<(&str, u32)>())
}
} }
} }