mii/tokenizers
1
0
Fork 0
mirror of https://github.com/mii443/tokenizers.git synced 2025-12-06 12:48:18 +00:00
Code Issues Packages Projects Releases Wiki Activity

Remove Container from Normalizers, replace with Arc.

Browse Source 
* prefix the Python types in Rust with Py
* remove unsound Container wrappers, replace with Arc
This commit is contained in:
Sebastian Pütz
2020-07-25 15:44:28 +02:00
committed by Anthony MOI
parent 83a52c8080
commit 08b8c48127
3 changed files with 120 additions and 143 deletions
Download Patch File Download Diff File Expand all files Collapse all files

18
bindings/python/src/lib.rs
View File

@@ -92,15 +92,15 @@ fn processors(_py: Python, m: &PyModule) -> PyResult<()> {
/// Normalizers Module
#[pymodule]
fn normalizers(_py: Python, m: &PyModule) -> PyResult<()> {
m.add_class::<normalizers::Normalizer>()?;
m.add_class::<normalizers::BertNormalizer>()?;
m.add_class::<normalizers::NFD>()?;
m.add_class::<normalizers::NFKD>()?;
m.add_class::<normalizers::NFC>()?;
m.add_class::<normalizers::NFKC>()?;
m.add_class::<normalizers::Sequence>()?;
m.add_class::<normalizers::Lowercase>()?;
m.add_class::<normalizers::Strip>()?;
m.add_class::<normalizers::PyNormalizer>()?;
m.add_class::<normalizers::PyBertNormalizer>()?;
m.add_class::<normalizers::PyNFD>()?;
m.add_class::<normalizers::PyNFKD>()?;
m.add_class::<normalizers::PyNFC>()?;
m.add_class::<normalizers::PyNFKC>()?;
m.add_class::<normalizers::PySequence>()?;
m.add_class::<normalizers::PyLowercase>()?;
m.add_class::<normalizers::PyStrip>()?;
Ok(())
}