Merge pull request #1316 from boyleconnor/add-expect-for-no-truncation

Add `expect()` for disabling truncation
This commit is contained in:
Arthur
2023-08-18 19:30:53 +02:00
committed by GitHub

View File

@ -718,7 +718,9 @@ impl PyTokenizer {
/// Disable truncation
#[pyo3(text_signature = "(self)")]
fn no_truncation(&mut self) {
let _ = self.tokenizer.with_truncation(None);
self.tokenizer
.with_truncation(None)
.expect("Failed to set truncation to `None`! This should never happen");
}
/// Get the currently set truncation parameters