mirror of
https://github.com/mii443/tokenizers.git
synced 2025-08-22 16:25:30 +00:00
linting
This commit is contained in:
@ -663,16 +663,19 @@ impl PyTokenizer {
|
|||||||
}
|
}
|
||||||
|
|
||||||
/// Get the underlying vocabulary
|
/// Get the underlying vocabulary
|
||||||
///
|
///
|
||||||
/// Returns:
|
/// Returns:
|
||||||
/// :obj:`Dict[int, AddedToken]`: The vocabulary
|
/// :obj:`Dict[int, AddedToken]`: The vocabulary
|
||||||
#[pyo3(signature = ())]
|
#[pyo3(signature = ())]
|
||||||
#[pyo3(text_signature = "(self)")]
|
#[pyo3(text_signature = "(self)")]
|
||||||
fn get_added_tokens_decoder(&self) -> HashMap<u32, PyAddedToken> {
|
fn get_added_tokens_decoder(&self) -> HashMap<u32, PyAddedToken> {
|
||||||
self.tokenizer.get_added_tokens_decoder().into_iter().map(|(key, value)| (key, value.into())).collect()
|
self.tokenizer
|
||||||
|
.get_added_tokens_decoder()
|
||||||
|
.into_iter()
|
||||||
|
.map(|(key, value)| (key, value.into()))
|
||||||
|
.collect()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
/// Get the size of the underlying vocabulary
|
/// Get the size of the underlying vocabulary
|
||||||
///
|
///
|
||||||
/// Args:
|
/// Args:
|
||||||
|
Reference in New Issue
Block a user