mii/tokenizers
1
0
Fork 0
mirror of https://github.com/mii443/tokenizers.git synced 2025-08-23 16:49:27 +00:00
Code Issues Packages Projects Releases Wiki Activity

Hide generics

Browse Source
This commit is contained in:
Sebastian Puetz
2020-07-31 12:22:13 +02:00
committed by Anthony MOI
parent d62adf7195
commit 42b810488f
9 changed files with 147 additions and 113 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
bindings/python/src/tokenizer.rs
View File

@ -7,7 +7,7 @@ use pyo3::types::*;
use pyo3::PyObjectProtocol;
use tk::models::bpe::BPE;
use tk::tokenizer::{
PaddingDirection, PaddingParams, PaddingStrategy, PostProcessor, Tokenizer, TruncationParams,
PaddingDirection, PaddingParams, PaddingStrategy, PostProcessor, TokenizerImpl, TruncationParams,
TruncationStrategy,
};
use tokenizers as tk;
@ -267,16 +267,16 @@ impl From<PreTokenizedEncodeInput> for tk::tokenizer::EncodeInput {
}
}
type TokenizerImpl = Tokenizer<PyModel, PyNormalizer, PyPreTokenizer, PyPostProcessor, PyDecoder>;
type Tokenizer = TokenizerImpl<PyModel, PyNormalizer, PyPreTokenizer, PyPostProcessor, PyDecoder>;
#[pyclass(dict, module = "tokenizers", name=Tokenizer)]
#[derive(Clone)]
pub struct PyTokenizer {
tokenizer: TokenizerImpl,
tokenizer: Tokenizer,
}
impl PyTokenizer {
fn new(tokenizer: TokenizerImpl) -> Self {
fn new(tokenizer: Tokenizer) -> Self {
PyTokenizer { tokenizer }
}
@ -331,7 +331,7 @@ impl PyTokenizer {
#[staticmethod]
fn from_file(path: &str) -> PyResult<Self> {
let tokenizer: PyResult<_> = ToPyResult(Tokenizer::from_file(path)).into();
let tokenizer: PyResult<_> = ToPyResult(TokenizerImpl::from_file(path)).into();
Ok(Self {
tokenizer: tokenizer?,
})