mii/tokenizers
1
0
Fork 0
mirror of https://github.com/mii443/tokenizers.git synced 2025-08-31 04:29:21 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #1316 from boyleconnor/add-expect-for-no-truncation

Browse Source 
Add `expect()` for disabling truncation
This commit is contained in:
Arthur
2023-08-18 19:30:53 +02:00
committed by GitHub
parent 540bf2eb01 748556a9ed
commit d0bb35d5a6
1 changed files with 3 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
bindings/python/src/tokenizer.rs
View File

@ -718,7 +718,9 @@ impl PyTokenizer {
/// Disable truncation /// Disable truncation
#[pyo3(text_signature = "(self)")] #[pyo3(text_signature = "(self)")]
fn no_truncation(&mut self) { fn no_truncation(&mut self) {
let _ = self.tokenizer.with_truncation(None); self.tokenizer
.with_truncation(None)
.expect("Failed to set truncation to `None`! This should never happen");
} }
/// Get the currently set truncation parameters /// Get the currently set truncation parameters