mii/tokenizers
1
0
Fork 0
mirror of https://github.com/mii443/tokenizers.git synced 2025-12-06 20:58:22 +00:00
Code Issues Packages Projects Releases Wiki Activity

python: fix inverted normalized/original string range

Browse Source
This commit is contained in:
Pierric Cistac
2020-01-31 11:09:55 -05:00
parent 6d532fedb1
commit 05275a9391
1 changed files with 2 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
bindings/python/src/encoding.rs
View File

@@ -77,8 +77,8 @@ impl PyMappingProtocol for IndexableString {
// Get the range from the relevant string
let s = match self.t {
IndexableStringType::Original => self.s.get_range(range),
IndexableStringType::Normalized => self.s.get_range_original(range),
IndexableStringType::Original => self.s.get_range_original(range),
IndexableStringType::Normalized => self.s.get_range(range),
};
s.map(|s| s.to_owned())