mii/tokenizers
1
0
Fork 0
mirror of https://github.com/mii443/tokenizers.git synced 2025-08-24 00:59:19 +00:00
Code Issues Packages Projects Releases Wiki Activity

Python - add missing modules exports

Browse Source
This commit is contained in:
Anthony MOI
2019-12-09 12:50:53 -05:00
parent 3979096c52
commit 849272d44f
3 changed files with 27 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
bindings/python/src/tokenizer.rs
View File

@ -9,7 +9,7 @@ use super::pre_tokenizers::PreTokenizer;
use super::token::Token;
use super::trainers::Trainer;
#[pyclass]
#[pyclass(dict)]
pub struct Tokenizer {
tokenizer: tk::tokenizer::Tokenizer,
}