mii/tokenizers
1
0
Fork 0
mirror of https://github.com/mii443/tokenizers.git synced 2025-08-23 00:35:35 +00:00
Code Issues Packages Projects Releases Wiki Activity

Python - Improve mappings on Encoding

Browse Source
This commit is contained in:
Anthony MOI
2020-04-14 22:03:12 -04:00
parent 3fb347b453
commit c5e22c14cb
2 changed files with 45 additions and 24 deletions
Download Patch File Download Diff File Expand all files Collapse all files

20
bindings/python/src/encoding.rs
View File

@ -96,20 +96,24 @@ impl Encoding {
.collect()
}
fn char_to_word_offsets(&self, pos: usize) -> Option<Offsets> {
self.encoding.char_to_word_offsets(pos)
fn word_to_tokens(&self, word_index: u32) -> Option<(usize, usize)> {
self.encoding.word_to_tokens(word_index)
}
fn char_to_token_offsets(&self, pos: usize) -> Option<Offsets> {
self.encoding.char_to_token_offsets(pos)
fn word_to_chars(&self, word_index: u32) -> Option<Offsets> {
self.encoding.word_to_chars(word_index)
}
fn token_to_word_offsets(&self, index: usize) -> Option<Offsets> {
self.encoding.token_to_word_offsets(index)
fn token_to_chars(&self, token_index: usize) -> Option<Offsets> {
self.encoding.token_to_chars(token_index)
}
fn char_to_token(&self, pos: usize) -> Option<usize> {
self.encoding.char_to_token(pos)
fn token_to_word(&self, token_index: usize) -> Option<u32> {
self.encoding.token_to_word(token_index)
}
fn char_to_token(&self, char_pos: usize) -> Option<usize> {
self.encoding.char_to_token(char_pos)
}
#[args(kwargs = "**")]