mii/tokenizers
1
0
Fork 0
mirror of https://github.com/mii443/tokenizers.git synced 2025-08-22 16:25:30 +00:00
Code Issues Packages Projects Releases Wiki Activity

Arg name correction: auth_token -> token (#1621)

Browse Source 
* Arg name correction: auth_token -> token

* Arg name correction in .rs: auth_token -> token

* update from_pretrained.rs file as well

---------

Co-authored-by: Rene Ravenel <rene@Renes-MacBook-Pro.local>
Co-authored-by: Arthur Zucker <arthur.zucker@gmail.com>
This commit is contained in:
rravenel
2024-10-24 07:32:09 -07:00
committed by GitHub
parent 9b77c054ef
commit a8738a95d1
3 changed files with 11 additions and 11 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
bindings/python/py_src/tokenizers/__init__.pyi
View File

@ -971,7 +971,7 @@ class Tokenizer:
pass
@staticmethod
def from_pretrained(identifier, revision="main", auth_token=None):
def from_pretrained(identifier, revision="main", token=None):
"""
Instantiate a new :class:`~tokenizers.Tokenizer` from an existing file on the
Hugging Face Hub.
@ -982,7 +982,7 @@ class Tokenizer:
a tokenizer.json file
revision (:obj:`str`, defaults to `main`):
A branch or commit id
auth_token (:obj:`str`, `optional`, defaults to `None`):
token (:obj:`str`, `optional`, defaults to `None`):
An optional auth token used to access private repositories on the
Hugging Face Hub

12
bindings/python/src/tokenizer.rs
View File

@ -578,19 +578,19 @@ impl PyTokenizer {
/// a tokenizer.json file
/// revision (:obj:`str`, defaults to `main`):
/// A branch or commit id
/// auth_token (:obj:`str`, `optional`, defaults to `None`):
/// token (:obj:`str`, `optional`, defaults to `None`):
/// An optional auth token used to access private repositories on the
/// Hugging Face Hub
///
/// Returns:
/// :class:`~tokenizers.Tokenizer`: The new tokenizer
#[staticmethod]
#[pyo3(signature = (identifier, revision = String::from("main"), auth_token = None))]
#[pyo3(text_signature = "(identifier, revision=\"main\", auth_token=None)")]
#[pyo3(signature = (identifier, revision = String::from("main"), token = None))]
#[pyo3(text_signature = "(identifier, revision=\"main\", token=None)")]
fn from_pretrained(
identifier: &str,
revision: String,
auth_token: Option<String>,
token: Option<String>,
) -> PyResult<Self> {
let path = Python::with_gil(|py| -> PyResult<String> {
let huggingface_hub = PyModule::import_bound(py, intern!(py, "huggingface_hub"))?;
@ -601,8 +601,8 @@ impl PyTokenizer {
(intern!(py, "revision"), &revision),
]
.into_py_dict_bound(py);
if let Some(auth_token) = auth_token {
kwargs.set_item(intern!(py, "token"), auth_token)?;
if let Some(token) = token {
kwargs.set_item(intern!(py, "token"), token)?;
}
let path: String = hf_hub_download.call((), Some(&kwargs))?.extract()?;
Ok(path)

6
tokenizers/src/utils/from_pretrained.rs
View File

@ -8,7 +8,7 @@ use std::path::PathBuf;
pub struct FromPretrainedParameters {
pub revision: String,
pub user_agent: HashMap<String, String>,
pub auth_token: Option<String>,
pub token: Option<String>,
}
impl Default for FromPretrainedParameters {
@ -16,7 +16,7 @@ impl Default for FromPretrainedParameters {
Self {
revision: "main".into(),
user_agent: HashMap::new(),
auth_token: None,
token: None,
}
}
}
@ -60,7 +60,7 @@ pub fn from_pretrained<S: AsRef<str>>(
}
let mut builder = ApiBuilder::new();
if let Some(token) = params.auth_token {
if let Some(token) = params.token {
builder = builder.with_token(Some(token));
}
let api = builder.build()?;