mirror of
https://github.com/mii443/tokenizers.git
synced 2025-08-22 16:25:30 +00:00
Fixing decoder strip because of char boundaries. (#1197)
This commit is contained in:
@ -23,7 +23,13 @@ impl Decoder for Strip {
|
||||
fn decode_chain(&self, tokens: Vec<String>) -> Result<Vec<String>> {
|
||||
Ok(tokens
|
||||
.into_iter()
|
||||
.map(|token| token[self.left..token.len() - self.right].to_string())
|
||||
.map(|token| {
|
||||
token
|
||||
.chars()
|
||||
.skip(self.left)
|
||||
.take(token.len() - self.left - self.right)
|
||||
.collect()
|
||||
})
|
||||
.collect())
|
||||
}
|
||||
}
|
||||
|
Reference in New Issue
Block a user