Python - Add pre_tokenizers module

bindings/python/src/lib.rs

@@ -1,4 +1,5 @@
 mod models;
+mod pre_tokenizers;
 mod token;
 mod tokenizer;
 mod utils;
@@ -14,10 +15,18 @@ pub fn models(_py: Python, m: &PyModule) -> PyResult<()> {
     Ok(())
 }
 
+/// PreTokenizers Module
+#[pymodule]
+pub fn pre_tokenizers(_py: Python, m: &PyModule) -> PyResult<()> {
+    m.add_class::<pre_tokenizers::ByteLevel>()?;
+    Ok(())
+}
+
 /// Tokenizers Module
 #[pymodule]
 fn tokenizers(_py: Python, m: &PyModule) -> PyResult<()> {
     m.add_class::<tokenizer::Tokenizer>()?;
     m.add_wrapped(wrap_pymodule!(models))?;
+    m.add_wrapped(wrap_pymodule!(pre_tokenizers))?;
     Ok(())
 }

bindings/python/src/pre_tokenizers.rs

@@ -0,0 +1,21 @@
+extern crate tokenizers as tk;
+
+use super::utils::Container;
+use pyo3::prelude::*;
+
+#[pyclass]
+pub struct PreTokenizer {
+    pub pretok: Container<dyn tk::tokenizer::PreTokenizer + Sync>,
+}
+
+#[pyclass]
+pub struct ByteLevel {}
+#[pymethods]
+impl ByteLevel {
+    #[staticmethod]
+    fn new() -> PyResult<PreTokenizer> {
+        Ok(PreTokenizer {
+            pretok: Container::Owned(Box::new(tk::pre_tokenizers::byte_level::ByteLevel)),
+        })
+    }
+}