mirror of
https://github.com/mii443/tokenizers.git
synced 2025-12-06 04:38:23 +00:00
Python - Update bindings
This commit is contained in:
@@ -159,6 +159,15 @@ impl Tokenizer {
|
||||
self.tokenizer.with_padding(None);
|
||||
}
|
||||
|
||||
fn normalize(&self, sentence: &str) -> PyResult<String> {
|
||||
ToPyResult(
|
||||
self.tokenizer
|
||||
.normalize(sentence)
|
||||
.map(|s| s.get().to_owned()),
|
||||
)
|
||||
.into()
|
||||
}
|
||||
|
||||
#[args(add_special_tokens = true)]
|
||||
fn encode(
|
||||
&self,
|
||||
|
||||
Reference in New Issue
Block a user