diff --git a/tokenizers/src/decoders/strip.rs b/tokenizers/src/decoders/strip.rs index 1691c707..0f6e0426 100644 --- a/tokenizers/src/decoders/strip.rs +++ b/tokenizers/src/decoders/strip.rs @@ -23,7 +23,13 @@ impl Decoder for Strip { fn decode_chain(&self, tokens: Vec) -> Result> { Ok(tokens .into_iter() - .map(|token| token[self.left..token.len() - self.right].to_string()) + .map(|token| { + token + .chars() + .skip(self.left) + .take(token.len() - self.left - self.right) + .collect() + }) .collect()) } }