mii/tokenizers
1
0
Fork 0
mirror of https://github.com/mii443/tokenizers.git synced 2025-08-22 16:25:30 +00:00
Code Issues Packages Projects Releases Wiki Activity

linting

Browse Source
This commit is contained in:
Arthur Zucker
2023-09-05 16:43:06 +00:00
parent 26fdfc2bc3
commit f435af8b71
1 changed files with 6 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
bindings/python/src/tokenizer.rs
View File

@ -669,10 +669,13 @@ impl PyTokenizer {
#[pyo3(signature = ())]
#[pyo3(text_signature = "(self)")]
fn get_added_tokens_decoder(&self) -> HashMap<u32, PyAddedToken> {
self.tokenizer.get_added_tokens_decoder().into_iter().map(|(key, value)| (key, value.into())).collect()
self.tokenizer
.get_added_tokens_decoder()
.into_iter()
.map(|(key, value)| (key, value.into()))
.collect()
}
/// Get the size of the underlying vocabulary
///
/// Args: