Python - Custom PreTokenizer backbone

bindings/python/src/pre_tokenizers.rs

@@ -7,6 +7,16 @@ use pyo3::prelude::*;
 pub struct PreTokenizer {
     pub pretok: Container<dyn tk::tokenizer::PreTokenizer + Sync>,
 }
+#[pymethods]
+impl PreTokenizer {
+    #[staticmethod]
+    fn from_python(pretok: PyObject) -> PyResult<Self> {
+        let py_pretok = PyPreTokenizer::new(pretok)?;
+        Ok(PreTokenizer {
+            pretok: Container::Owned(Box::new(py_pretok)),
+        })
+    }
+}
 
 #[pyclass]
 pub struct ByteLevel {}
@@ -19,3 +29,21 @@ impl ByteLevel {
         })
     }
 }
+
+/// Attempt at providing Python the ability to give its own PreTokenizer
+struct PyPreTokenizer {
+    class: PyObject,
+}
+
+impl PyPreTokenizer {
+    pub fn new(class: PyObject) -> PyResult<Self> {
+        // test the given PyObject
+        Ok(PyPreTokenizer { class })
+    }
+}
+
+impl tk::tokenizer::PreTokenizer for PyPreTokenizer {
+    fn pre_tokenize(&self, sentence: &str) -> Vec<String> {
+        unimplemented!()
+    }
+}