Python - Expose PostProcessors

bindings/python/src/tokenizer.rs

@@ -9,6 +9,7 @@ use super::encoding::Encoding;
 use super::error::ToPyResult;
 use super::models::Model;
 use super::pre_tokenizers::PreTokenizer;
+use super::processors::PostProcessor;
 use super::trainers::Trainer;
 
 #[pyclass(dict)]
@@ -69,6 +70,17 @@ impl Tokenizer {
         }
     }
 
+    fn with_post_processor(&mut self, processor: &mut PostProcessor) -> PyResult<()> {
+        if let Some(processor) = processor.processor.to_pointer() {
+            self.tokenizer.with_post_processor(processor);
+            Ok(())
+        } else {
+            Err(exceptions::Exception::py_err(
+                "The Processor is already being used in another Tokenizer",
+            ))
+        }
+    }
+
     fn encode(&self, sentence: &str, pair: Option<&str>) -> PyResult<Encoding> {
         ToPyResult(
             self.tokenizer