Train Model in place

This let us keep everything that was set on the model except from the vocabulary when trained. For example, this let us keep the configured `unk_token` of BPE when its trained.
This commit is contained in:
Anthony MOI
2020-10-08 18:20:38 -04:00
committed by Anthony MOI
parent 224862fe0c
commit 54c7210b2f
12 changed files with 156 additions and 121 deletions

View File

@ -1044,7 +1044,7 @@ impl PyTokenizer {
let trainer =
trainer.map_or_else(|| self.tokenizer.get_model().get_trainer(), |t| t.clone());
Python::with_gil(|py| {
py.allow_threads(|| ToPyResult(self.tokenizer.train_and_replace(&trainer, files)).into())
py.allow_threads(|| ToPyResult(self.tokenizer.train(&trainer, files)).into())
})
}