mii/tokenizers
1
0
Fork 0
mirror of https://github.com/mii443/tokenizers.git synced 2025-08-22 16:25:30 +00:00
Code Issues Packages Projects Releases Wiki Activity

Arg name correction: auth_token -> token (#1621)

Browse Source 
* Arg name correction: auth_token -> token

* Arg name correction in .rs: auth_token -> token

* update from_pretrained.rs file as well

---------

Co-authored-by: Rene Ravenel <rene@Renes-MacBook-Pro.local>
Co-authored-by: Arthur Zucker <arthur.zucker@gmail.com>
This commit is contained in:
rravenel
2024-10-24 07:32:09 -07:00
committed by GitHub
parent 9b77c054ef
commit a8738a95d1
3 changed files with 11 additions and 11 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
bindings/python/py_src/tokenizers/__init__.pyi
View File

@ -971,7 +971,7 @@ class Tokenizer:
pass pass
@staticmethod @staticmethod
def from_pretrained(identifier, revision="main", auth_token=None): def from_pretrained(identifier, revision="main", token=None):
""" """
Instantiate a new :class:`~tokenizers.Tokenizer` from an existing file on the Instantiate a new :class:`~tokenizers.Tokenizer` from an existing file on the
Hugging Face Hub. Hugging Face Hub.
@ -982,7 +982,7 @@ class Tokenizer:
a tokenizer.json file a tokenizer.json file
revision (:obj:`str`, defaults to `main`): revision (:obj:`str`, defaults to `main`):
A branch or commit id A branch or commit id
auth_token (:obj:`str`, `optional`, defaults to `None`): token (:obj:`str`, `optional`, defaults to `None`):
An optional auth token used to access private repositories on the An optional auth token used to access private repositories on the
Hugging Face Hub Hugging Face Hub

12
bindings/python/src/tokenizer.rs
View File

@ -578,19 +578,19 @@ impl PyTokenizer {
/// a tokenizer.json file /// a tokenizer.json file
/// revision (:obj:`str`, defaults to `main`): /// revision (:obj:`str`, defaults to `main`):
/// A branch or commit id /// A branch or commit id
/// auth_token (:obj:`str`, `optional`, defaults to `None`): /// token (:obj:`str`, `optional`, defaults to `None`):
/// An optional auth token used to access private repositories on the /// An optional auth token used to access private repositories on the
/// Hugging Face Hub /// Hugging Face Hub
/// ///
/// Returns: /// Returns:
/// :class:`~tokenizers.Tokenizer`: The new tokenizer /// :class:`~tokenizers.Tokenizer`: The new tokenizer
#[staticmethod] #[staticmethod]
#[pyo3(signature = (identifier, revision = String::from("main"), auth_token = None))] #[pyo3(signature = (identifier, revision = String::from("main"), token = None))]
#[pyo3(text_signature = "(identifier, revision=\"main\", auth_token=None)")] #[pyo3(text_signature = "(identifier, revision=\"main\", token=None)")]
fn from_pretrained( fn from_pretrained(
identifier: &str, identifier: &str,
revision: String, revision: String,
auth_token: Option<String>, token: Option<String>,
) -> PyResult<Self> { ) -> PyResult<Self> {
let path = Python::with_gil(|py| -> PyResult<String> { let path = Python::with_gil(|py| -> PyResult<String> {
let huggingface_hub = PyModule::import_bound(py, intern!(py, "huggingface_hub"))?; let huggingface_hub = PyModule::import_bound(py, intern!(py, "huggingface_hub"))?;
@ -601,8 +601,8 @@ impl PyTokenizer {
(intern!(py, "revision"), &revision), (intern!(py, "revision"), &revision),
] ]
.into_py_dict_bound(py); .into_py_dict_bound(py);
if let Some(auth_token) = auth_token { if let Some(token) = token {
kwargs.set_item(intern!(py, "token"), auth_token)?; kwargs.set_item(intern!(py, "token"), token)?;
} }
let path: String = hf_hub_download.call((), Some(&kwargs))?.extract()?; let path: String = hf_hub_download.call((), Some(&kwargs))?.extract()?;
Ok(path) Ok(path)

6
tokenizers/src/utils/from_pretrained.rs
View File

@ -8,7 +8,7 @@ use std::path::PathBuf;
pub struct FromPretrainedParameters { pub struct FromPretrainedParameters {
pub revision: String, pub revision: String,
pub user_agent: HashMap<String, String>, pub user_agent: HashMap<String, String>,
pub auth_token: Option<String>, pub token: Option<String>,
} }
impl Default for FromPretrainedParameters { impl Default for FromPretrainedParameters {
@ -16,7 +16,7 @@ impl Default for FromPretrainedParameters {
Self { Self {
revision: "main".into(), revision: "main".into(),
user_agent: HashMap::new(), user_agent: HashMap::new(),
auth_token: None, token: None,
} }
} }
} }
@ -60,7 +60,7 @@ pub fn from_pretrained<S: AsRef<str>>(
} }
let mut builder = ApiBuilder::new(); let mut builder = ApiBuilder::new();
if let Some(token) = params.auth_token { if let Some(token) = params.token {
builder = builder.with_token(Some(token)); builder = builder.with_token(Some(token));
} }
let api = builder.build()?; let api = builder.build()?;