mii/tokenizers
1
0
Fork 0
mirror of https://github.com/mii443/tokenizers.git synced 2025-08-23 00:35:35 +00:00
Code Issues Packages Projects Releases Wiki Activity

Python - Update mappings API

Browse Source
This commit is contained in:
Anthony MOI
2020-03-23 21:37:44 -04:00
parent e9babc8383
commit 1150751ab6
2 changed files with 49 additions and 16 deletions
Download Patch File Download Diff File Expand all files Collapse all files

18
bindings/python/src/encoding.rs
View File

@ -83,22 +83,22 @@ impl Encoding {
.collect()
}
fn word_boundaries(&self, index: usize) -> Option<(usize, usize)> {
self.encoding.word_boundaries(index)
fn char_to_word_offsets(&self, pos: usize) -> Option<Offsets> {
self.encoding.char_to_word_offsets(pos)
}
fn char_to_word(&self, pos: usize) -> Option<Offsets> {
self.encoding.char_to_word(pos)
fn char_to_token_offsets(&self, pos: usize) -> Option<Offsets> {
self.encoding.char_to_token_offsets(pos)
}
fn char_to_token(&self, pos: usize) -> Option<Offsets> {
fn token_to_word_offsets(&self, index: usize) -> Option<Offsets> {
self.encoding.token_to_word_offsets(index)
}
fn char_to_token(&self, pos: usize) -> Option<usize> {
self.encoding.char_to_token(pos)
}
fn token_to_word(&self, index: usize) -> Option<Offsets> {
self.encoding.token_to_word(index)
}
#[args(kwargs = "**")]
fn pad(&mut self, length: usize, kwargs: Option<&PyDict>) -> PyResult<()> {
let mut pad_id = 0;