mii/tokenizers
1
0
Fork 0
mirror of https://github.com/mii443/tokenizers.git synced 2025-08-23 00:35:35 +00:00
Code Issues Packages Projects Releases Wiki Activity

Add ability to train from Iterator

Browse Source
This commit is contained in:
Anthony MOI
2020-11-12 12:58:14 -05:00
committed by Anthony MOI
parent 6e364cb685
commit e0a70f1fb2
11 changed files with 380 additions and 199 deletions
Download Patch File Download Diff File Expand all files Collapse all files

18
bindings/python/src/trainers.rs
View File

@ -1,4 +1,3 @@
use std::collections::HashMap;
use std::sync::{Arc, RwLock};
use pyo3::exceptions;
@ -50,19 +49,20 @@ impl Trainer for PyTrainer {
self.trainer.read().unwrap().should_show_progress()
}
fn train(
&self,
words: HashMap<String, u32>,
model: &mut PyModel,
) -> tk::Result<Vec<tk::AddedToken>> {
fn train(&self, model: &mut PyModel) -> tk::Result<Vec<tk::AddedToken>> {
self.trainer
.read()
.unwrap()
.train(words, &mut model.model.write().unwrap())
.train(&mut model.model.write().unwrap())
}
fn process_tokens(&self, words: &mut HashMap<String, u32>, tokens: Vec<String>) {
self.trainer.read().unwrap().process_tokens(words, tokens)
fn feed<I, S, F>(&mut self, iterator: I, process: F) -> tk::Result<()>
where
I: Iterator<Item = S> + Send,
S: AsRef<str> + Send,
F: Fn(&str) -> tk::Result<Vec<String>> + Sync,
{
self.trainer.write().unwrap().feed(iterator, process)
}
}