Python - Update bindings

This commit is contained in:
Anthony MOI
2020-03-16 10:28:34 -04:00
parent e0cfad5102
commit 60a4fb35f4
3 changed files with 22 additions and 173 deletions

View File

@@ -159,6 +159,15 @@ impl Tokenizer {
self.tokenizer.with_padding(None);
}
fn normalize(&self, sentence: &str) -> PyResult<String> {
ToPyResult(
self.tokenizer
.normalize(sentence)
.map(|s| s.get().to_owned()),
)
.into()
}
#[args(add_special_tokens = true)]
fn encode(
&self,