mii/tokenizers
1
0
Fork 0
mirror of https://github.com/mii443/tokenizers.git synced 2025-12-06 04:38:23 +00:00
Code Issues Packages Projects Releases Wiki Activity

Replace Model and Trainer Containers.

Browse Source 
* Implement changes necessary from generic Model in Tokenizer.
* Temporarily disable training in Python since Clone can't be
  derived for Model until all components have been replaced.
* Prefix Python types in Rust with Py.
This commit is contained in:
Sebastian Pütz
2020-07-25 03:57:16 +02:00
committed by Anthony MOI
parent cdef780aa8
commit 83a52c8080
17 changed files with 360 additions and 299 deletions
Download Patch File Download Diff File Expand all files Collapse all files

20
bindings/python/src/lib.rs
View File

@@ -39,19 +39,19 @@ extern "C" fn child_after_fork() {
/// Trainers Module
#[pymodule]
fn trainers(_py: Python, m: &PyModule) -> PyResult<()> {
m.add_class::<trainers::Trainer>()?;
m.add_class::<trainers::BpeTrainer>()?;
m.add_class::<trainers::WordPieceTrainer>()?;
m.add_class::<trainers::PyTrainer>()?;
m.add_class::<trainers::PyBpeTrainer>()?;
m.add_class::<trainers::PyWordPieceTrainer>()?;
Ok(())
}
/// Models Module
#[pymodule]
fn models(_py: Python, m: &PyModule) -> PyResult<()> {
m.add_class::<models::Model>()?;
m.add_class::<models::BPE>()?;
m.add_class::<models::WordPiece>()?;
m.add_class::<models::WordLevel>()?;
m.add_class::<models::PyModel>()?;
m.add_class::<models::PyBPE>()?;
m.add_class::<models::PyWordPiece>()?;
m.add_class::<models::PyWordLevel>()?;
Ok(())
}
@@ -116,9 +116,9 @@ fn tokenizers(_py: Python, m: &PyModule) -> PyResult<()> {
}
}
m.add_class::<tokenizer::Tokenizer>()?;
m.add_class::<tokenizer::AddedToken>()?;
m.add_class::<encoding::Encoding>()?;
m.add_class::<tokenizer::PyTokenizer>()?;
m.add_class::<tokenizer::PyAddedToken>()?;
m.add_class::<encoding::PyEncoding>()?;
m.add_wrapped(wrap_pymodule!(models))?;
m.add_wrapped(wrap_pymodule!(pre_tokenizers))?;
m.add_wrapped(wrap_pymodule!(decoders))?;