Expose Tokenizer parts

bindings/python/src/models.rs

@@ -5,7 +5,7 @@ use super::utils::Container;
 use pyo3::exceptions;
 use pyo3::prelude::*;
 use pyo3::types::*;
-use std::path::{Path, PathBuf};
+use std::path::Path;
 
 /// A Model represents some tokenization algorithm like BPE or Word
 /// This class cannot be constructed directly. Please use one of the concrete models.

bindings/python/src/tokenizer.rs

@@ -12,6 +12,7 @@ use super::normalizers::Normalizer;
 use super::pre_tokenizers::PreTokenizer;
 use super::processors::PostProcessor;
 use super::trainers::Trainer;
+use super::utils::Container;
 
 use tk::tokenizer::{
     PaddingDirection, PaddingParams, PaddingStrategy, TruncationParams, TruncationStrategy,
@@ -301,4 +302,48 @@ impl Tokenizer {
             }
         })
     }
+
+    #[getter]
+    fn get_model(&self) -> PyResult<Model> {
+        Ok(Model {
+            model: Container::from_ref(self.tokenizer.get_model()),
+        })
+    }
+
+    #[getter]
+    fn get_normalizer(&self) -> PyResult<Option<Normalizer>> {
+        Ok(self
+            .tokenizer
+            .get_normalizer()
+            .map(|normalizer| Normalizer {
+                normalizer: Container::from_ref(normalizer),
+            }))
+    }
+
+    #[getter]
+    fn get_pre_tokenizer(&self) -> PyResult<Option<PreTokenizer>> {
+        Ok(self
+            .tokenizer
+            .get_pre_tokenizer()
+            .map(|pretok| PreTokenizer {
+                pretok: Container::from_ref(pretok),
+            }))
+    }
+
+    #[getter]
+    fn get_post_processor(&self) -> PyResult<Option<PostProcessor>> {
+        Ok(self
+            .tokenizer
+            .get_post_processor()
+            .map(|processor| PostProcessor {
+                processor: Container::from_ref(processor),
+            }))
+    }
+
+    #[getter]
+    fn get_decoder(&self) -> PyResult<Option<Decoder>> {
+        Ok(self.tokenizer.get_decoder().map(|decoder| Decoder {
+            decoder: Container::from_ref(decoder),
+        }))
+    }
 }

bindings/python/src/utils.rs

@@ -17,6 +17,11 @@ impl<T> Container<T>
 where
     T: ?Sized,
 {
+    pub fn from_ref(reference: &Box<T>) -> Self {
+        let content: *const T = &**reference;
+        Container::Pointer(content as *mut _)
+    }
+
     /// Consumes ourself and return the Boxed element if we have the ownership, None otherwise.
     pub fn take(self) -> Option<Box<T>> {
         match self {