Add Tokenizer token_to_id/id_to_token

This commit is contained in:
Anthony MOI
2019-11-20 17:28:28 -05:00
parent 8b3d7d1aa0
commit 3ec26b332c
3 changed files with 28 additions and 0 deletions

View File

@@ -153,6 +153,14 @@ impl Tokenizer {
})
.collect()
}
fn token_to_id(&self, token: &str) -> Option<u32> {
self.tokenizer.token_to_id(token)
}
fn id_to_token(&self, id: u32) -> Option<String> {
self.tokenizer.id_to_token(id)
}
}
#[pymodule]