mii/tokenizers
1
0
Fork 0
mirror of https://github.com/mii443/tokenizers.git synced 2025-08-22 16:25:30 +00:00
Code Issues Packages Projects Releases Wiki Activity

Improve some Python classes

Browse Source
This commit is contained in:
Anthony MOI
2019-12-13 16:35:25 -05:00
parent 4914e6285e
commit 24139d7324
2 changed files with 2 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
bindings/python/src/decoders.rs
View File

@ -6,7 +6,7 @@ use pyo3::prelude::*;
use pyo3::types::*;
use tk::tokenizer::Result;
#[pyclass]
#[pyclass(dict)]
pub struct Decoder {
pub decoder: Container<dyn tk::tokenizer::Decoder + Sync>,
}

2
bindings/python/src/pre_tokenizers.rs
View File

@ -7,7 +7,7 @@ use pyo3::types::*;
use std::collections::HashSet;
use tk::tokenizer::Result;
#[pyclass]
#[pyclass(dict)]
pub struct PreTokenizer {
pub pretok: Container<dyn tk::tokenizer::PreTokenizer + Sync>,
}