This commit is contained in:
Arthur Zucker
2023-09-05 16:43:06 +00:00
parent 26fdfc2bc3
commit f435af8b71

View File

@ -663,16 +663,19 @@ impl PyTokenizer {
} }
/// Get the underlying vocabulary /// Get the underlying vocabulary
/// ///
/// Returns: /// Returns:
/// :obj:`Dict[int, AddedToken]`: The vocabulary /// :obj:`Dict[int, AddedToken]`: The vocabulary
#[pyo3(signature = ())] #[pyo3(signature = ())]
#[pyo3(text_signature = "(self)")] #[pyo3(text_signature = "(self)")]
fn get_added_tokens_decoder(&self) -> HashMap<u32, PyAddedToken> { fn get_added_tokens_decoder(&self) -> HashMap<u32, PyAddedToken> {
self.tokenizer.get_added_tokens_decoder().into_iter().map(|(key, value)| (key, value.into())).collect() self.tokenizer
.get_added_tokens_decoder()
.into_iter()
.map(|(key, value)| (key, value.into()))
.collect()
} }
/// Get the size of the underlying vocabulary /// Get the size of the underlying vocabulary
/// ///
/// Args: /// Args: