mii/tokenizers
1
0
Fork 0
mirror of https://github.com/mii443/tokenizers.git synced 2025-08-23 00:35:35 +00:00
Code Issues Packages Projects Releases Wiki Activity

Testing ABI3 wheels to reduce number of wheels (#1674)

Browse Source 
* Testing ABI3 wheels to reduce number of wheels

* No need for py-clone  anymore.

* Upgrade python versions.

* Remove those flakes.

* Promoting new CI + Fixing secret.
This commit is contained in:
Nicolas Patry
2024-11-15 13:02:22 +08:00
committed by GitHub
parent 5aa9f6cff0
commit f4c9fd7f40
14 changed files with 246 additions and 66 deletions
Download Patch File Download Diff File Expand all files Collapse all files

16
bindings/python/src/trainers.rs
View File

@ -313,8 +313,8 @@ impl PyBpeTrainer {
let mut builder = tk::models::bpe::BpeTrainer::builder();
if let Some(kwargs) = kwargs {
for (key, val) in kwargs {
let key: &str = key.extract()?;
match key {
let key: String = key.extract()?;
match key.as_ref() {
"vocab_size" => builder = builder.vocab_size(val.extract()?),
"min_frequency" => builder = builder.min_frequency(val.extract()?),
"show_progress" => builder = builder.show_progress(val.extract()?),
@ -520,8 +520,8 @@ impl PyWordPieceTrainer {
let mut builder = tk::models::wordpiece::WordPieceTrainer::builder();
if let Some(kwargs) = kwargs {
for (key, val) in kwargs {
let key: &str = key.extract()?;
match key {
let key: String = key.extract()?;
match key.as_ref() {
"vocab_size" => builder = builder.vocab_size(val.extract()?),
"min_frequency" => builder = builder.min_frequency(val.extract()?),
"show_progress" => builder = builder.show_progress(val.extract()?),
@ -661,8 +661,8 @@ impl PyWordLevelTrainer {
if let Some(kwargs) = kwargs {
for (key, val) in kwargs {
let key: &str = key.extract()?;
match key {
let key: String = key.extract()?;
match key.as_ref() {
"vocab_size" => {
builder.vocab_size(val.extract()?);
}
@ -828,8 +828,8 @@ impl PyUnigramTrainer {
let mut builder = tk::models::unigram::UnigramTrainer::builder();
if let Some(kwargs) = kwargs {
for (key, val) in kwargs {
let key: &str = key.extract()?;
match key {
let key: String = key.extract()?;
match key.as_ref() {
"vocab_size" => builder.vocab_size(val.extract()?),
"show_progress" => builder.show_progress(val.extract()?),
"n_sub_iterations" => builder.n_sub_iterations(val.extract()?),