mirror of
https://github.com/mii443/tokenizers.git
synced 2025-12-06 04:38:23 +00:00
Add Tokenizer token_to_id/id_to_token
This commit is contained in:
@@ -153,6 +153,14 @@ impl Tokenizer {
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn token_to_id(&self, token: &str) -> Option<u32> {
|
||||
self.tokenizer.token_to_id(token)
|
||||
}
|
||||
|
||||
fn id_to_token(&self, id: u32) -> Option<String> {
|
||||
self.tokenizer.id_to_token(id)
|
||||
}
|
||||
}
|
||||
|
||||
#[pymodule]
|
||||
|
||||
Reference in New Issue
Block a user