mii/tokenizers
1
0
Fork 0
mirror of https://github.com/mii443/tokenizers.git synced 2025-08-31 12:39:21 +00:00
Code Issues Packages Projects Releases Wiki Activity

python: fix inverted normalized/original string range

Browse Source
This commit is contained in:
Pierric Cistac
2020-01-31 11:09:55 -05:00
parent 6d532fedb1
commit 05275a9391
1 changed files with 2 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
bindings/python/src/encoding.rs
View File

@ -77,8 +77,8 @@ impl PyMappingProtocol for IndexableString {
// Get the range from the relevant string // Get the range from the relevant string
let s = match self.t { let s = match self.t {
IndexableStringType::Original => self.s.get_range(range), IndexableStringType::Original => self.s.get_range_original(range),
IndexableStringType::Normalized => self.s.get_range_original(range), IndexableStringType::Normalized => self.s.get_range(range),
}; };
s.map(|s| s.to_owned()) s.map(|s| s.to_owned())