mii/tokenizers
1
0
Fork 0
mirror of https://github.com/mii443/tokenizers.git synced 2025-08-23 16:49:27 +00:00
Code Issues Packages Projects Releases Wiki Activity

Python - Add missing Bpe constructor kwargs

Browse Source
This commit is contained in:
Anthony MOI
2020-01-06 16:39:59 -05:00
parent b7d0acc562
commit 9a99e2bcb1
1 changed files with 4 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
bindings/python/src/models.rs
View File

@ -62,6 +62,10 @@ impl BPE {
"cache_capacity" => builder = builder.cache_capacity(value.extract()?),
"dropout" => builder = builder.dropout(value.extract()?),
"unk_token" => builder = builder.unk_token(value.extract()?),
"continuing_subword_prefix" => {
builder = builder.continuing_subword_prefix(value.extract()?)
}
"end_of_word_suffix" => builder = builder.end_of_word_suffix(value.extract()?),
_ => println!("Ignored unknown kwarg option {}", key),
};
}