mii/tokenizers
1
0
Fork 0
mirror of https://github.com/mii443/tokenizers.git synced 2025-08-22 16:25:30 +00:00
Code Issues Packages Projects Releases Wiki Activity

linting

Browse Source
This commit is contained in:
Arthur Zucker
2023-09-05 16:43:06 +00:00
parent 26fdfc2bc3
commit f435af8b71
1 changed files with 6 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
bindings/python/src/tokenizer.rs
View File

@ -663,16 +663,19 @@ impl PyTokenizer {
}
/// Get the underlying vocabulary
///
///
/// Returns:
/// :obj:`Dict[int, AddedToken]`: The vocabulary
#[pyo3(signature = ())]
#[pyo3(text_signature = "(self)")]
fn get_added_tokens_decoder(&self) -> HashMap<u32, PyAddedToken> {
self.tokenizer.get_added_tokens_decoder().into_iter().map(|(key, value)| (key, value.into())).collect()
self.tokenizer
.get_added_tokens_decoder()
.into_iter()
.map(|(key, value)| (key, value.into()))
.collect()
}
/// Get the size of the underlying vocabulary
///
/// Args: