mii/tokenizers
1
0
Fork 0
mirror of https://github.com/mii443/tokenizers.git synced 2025-09-01 14:59:20 +00:00
Code Issues Packages Projects Releases Wiki Activity

Handle special tokens

Browse Source
This commit is contained in:
Anthony MOI
2019-12-19 19:48:16 -05:00
parent 7f032b62df
commit a8d68d516d
2 changed files with 24 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
bindings/python/src/tokenizer.rs
View File

@ -233,6 +233,10 @@ impl Tokenizer {
Ok(self.tokenizer.add_tokens(&tokens))
}
fn add_special_tokens(&mut self, tokens: Vec<&str>) -> PyResult<usize> {
Ok(self.tokenizer.add_special_tokens(&tokens))
}
fn train(&mut self, trainer: &Trainer, files: Vec<String>) -> PyResult<()> {
trainer.trainer.execute(|trainer| {
if let Err(e) = self.tokenizer.train(trainer, files) {