mii/tokenizers
1
0
Fork 0
mirror of https://github.com/mii443/tokenizers.git synced 2025-08-23 00:35:35 +00:00
Code Issues Packages Projects Releases Wiki Activity

PyModel uses a RwLock to allow modifications

Browse Source
This commit is contained in:
Anthony MOI
2020-10-08 19:33:30 -04:00
committed by Anthony MOI
parent 54c7210b2f
commit 284a1dbee7
10 changed files with 76 additions and 69 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
bindings/python/src/trainers.rs
View File

@ -89,8 +89,7 @@ impl Trainer for PyTrainer {
words: HashMap<String, u32>,
model: &mut PyModel,
) -> tk::Result<Vec<tk::AddedToken>> {
todo!("FIX THIS");
self.trainer.train(words, &mut model.model)
self.trainer.train(words, &mut model.model.write().unwrap())
}
fn process_tokens(&self, words: &mut HashMap<String, u32>, tokens: Vec<String>) {