mii/tokenizers
1
0
Fork 0
mirror of https://github.com/mii443/tokenizers.git synced 2025-08-23 00:35:35 +00:00
Code Issues Packages Projects Releases Wiki Activity

Python - Decoders can get/set their attributes

Browse Source
This commit is contained in:
Anthony MOI
2020-11-23 22:41:27 -05:00
committed by Anthony MOI
parent 091287dcf5
commit 5c35fafc44
8 changed files with 159 additions and 16 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
bindings/python/src/normalizers.rs
View File

@ -3,6 +3,7 @@ use std::sync::{Arc, RwLock};
use pyo3::exceptions;
use pyo3::prelude::*;
use pyo3::types::*;
use pyo3::PySequenceProtocol;
use crate::error::ToPyResult;
use crate::utils::{PyNormalizedString, PyNormalizedStringRefMut, PyPattern};
@ -341,6 +342,13 @@ impl PySequence {
}
}
#[pyproto]
impl PySequenceProtocol for PySequence {
fn __len__(&self) -> usize {
0
}
}
/// Lowercase Normalizer
#[pyclass(extends=PyNormalizer, module = "tokenizers.normalizers", name=Lowercase)]
#[text_signature = "(self)"]