Python - Add decoders

bindings/python/src/tokenizer.rs

@@ -3,6 +3,7 @@ extern crate tokenizers as tk;
 use pyo3::exceptions;
 use pyo3::prelude::*;
 
+use super::decoders::Decoder;
 use super::models::Model;
 use super::pre_tokenizers::PreTokenizer;
 use super::token::Token;
@@ -49,6 +50,17 @@ impl Tokenizer {
         }
     }
 
+    fn with_decoder(&mut self, decoder: &mut Decoder) -> PyResult<()> {
+        if let Some(decoder) = decoder.decoder.to_pointer() {
+            self.tokenizer.with_decoder(decoder);
+            Ok(())
+        } else {
+            Err(exceptions::Exception::py_err(
+                "The Decoder is already being used in another Tokenizer",
+            ))
+        }
+    }
+
     fn encode(&self, sentence: &str) -> Vec<Token> {
         self.tokenizer
             .encode(sentence)