Expose post_process on the Tokenizer

This commit is contained in:
Anthony MOI
2020-03-25 17:54:48 -04:00
parent 9ce895550b
commit 9bd9e0b3c1
6 changed files with 126 additions and 2 deletions

View File

@ -343,6 +343,25 @@ impl Tokenizer {
})
}
#[args(pair = "None", add_special_tokens = true)]
fn post_process(
&self,
encoding: &Encoding,
pair: Option<&Encoding>,
add_special_tokens: bool,
) -> PyResult<Encoding> {
ToPyResult(
self.tokenizer
.post_process(
encoding.encoding.clone(),
pair.map(|p| p.encoding.clone()),
add_special_tokens,
)
.map(Encoding::new),
)
.into()
}
#[getter]
fn get_model(&self) -> PyResult<Model> {
Ok(Model {