mii/tokenizers
1
0
Fork 0
mirror of https://github.com/mii443/tokenizers.git synced 2025-08-23 00:35:35 +00:00
Code Issues Packages Projects Releases Wiki Activity

Python - Add missing char_to_word

Browse Source
This commit is contained in:
Anthony MOI
2020-04-16 18:20:30 -04:00
parent bdfb02f473
commit 873ac2d9a8
1 changed files with 4 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
bindings/python/src/encoding.rs
View File

@ -116,6 +116,10 @@ impl Encoding {
self.encoding.char_to_token(char_pos)
}
fn char_to_word(&self, char_pos: usize) -> Option<u32> {
self.encoding.char_to_word(char_pos)
}
#[args(kwargs = "**")]
fn pad(&mut self, length: usize, kwargs: Option<&PyDict>) -> PyResult<()> {
let mut pad_id = 0;