mii/tokenizers
1
0
Fork 0
mirror of https://github.com/mii443/tokenizers.git synced 2025-08-23 08:45:38 +00:00
Code Issues Packages Projects Releases Wiki Activity

Python - Update bindings

Browse Source
This commit is contained in:
Anthony MOI
2020-08-17 18:03:25 -05:00
committed by Anthony MOI
parent ed9da7731a
commit f92c9955e7
5 changed files with 55 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
bindings/python/src/tokenizer.rs
View File

@ -553,7 +553,7 @@ impl PyTokenizer {
ToPyResult(
self.tokenizer
.encode(input, add_special_tokens)
.encode_char_offsets(input, add_special_tokens)
.map(|e| e.into()),
)
.into()
@ -588,7 +588,7 @@ impl PyTokenizer {
gil.python().allow_threads(|| {
ToPyResult(
self.tokenizer
.encode_batch(input, add_special_tokens)
.encode_batch_char_offsets(input, add_special_tokens)
.map(|encodings| encodings.into_iter().map(|e| e.into()).collect()),
)
.into()