mii/tokenizers
1
0
Fork 0
mirror of https://github.com/mii443/tokenizers.git synced 2025-08-22 16:25:30 +00:00
Code Issues Packages Projects Releases Wiki Activity

Better export for FromPretrainedParameters

Browse Source
This commit is contained in:
Anthony Moi
2021-08-19 15:14:49 +02:00
committed by Anthony MOI
parent e44fdee4a1
commit 6f9e867330
2 changed files with 4 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
bindings/python/src/tokenizer.rs
View File

@ -567,7 +567,7 @@ impl PyTokenizer {
revision: String,
auth_token: Option<String>,
) -> PyResult<Self> {
let params = tk::utils::from_pretrained::FromPretrainedParameters {
let params = tk::FromPretrainedParameters {
revision,
auth_token,
user_agent: [("bindings", "Python"), ("version", crate::VERSION)]

3
tokenizers/src/lib.rs
View File

@ -129,3 +129,6 @@ pub use tokenizer::*;
// Re-export also parallelism utils
pub use utils::parallelism;
// Re-export for from_pretrained
pub use utils::from_pretrained::FromPretrainedParameters;