mii/tokenizers
1
0
Fork 0
mirror of https://github.com/mii443/tokenizers.git synced 2025-08-23 00:35:35 +00:00
Code Issues Packages Projects Releases Wiki Activity

Fixing tokenizers with 1.53 (updated some dependencies + clippy) (#764)

Browse Source
This commit is contained in:
Nicolas Patry
2021-07-21 09:58:38 +02:00
committed by GitHub
parent 755e5f5c1e
commit d83772d62c
5 changed files with 461 additions and 352 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
bindings/python/src/trainers.rs
View File

@ -291,8 +291,7 @@ impl PyBpeTrainer {
alphabet
.into_iter()
.map(|s| s.chars().next())
.filter(|c| c.is_some())
.map(|c| c.unwrap())
.flatten()
.collect(),
);
}
@ -497,8 +496,7 @@ impl PyWordPieceTrainer {
alphabet
.into_iter()
.map(|s| s.chars().next())
.filter(|c| c.is_some())
.map(|c| c.unwrap())
.flatten()
.collect(),
);
}
@ -773,8 +771,7 @@ impl PyUnigramTrainer {
alphabet
.into_iter()
.map(|s| s.chars().next())
.filter(|c| c.is_some())
.map(|c| c.unwrap())
.flatten()
.collect(),
)
}