From c205afe7a55c9631fa1b27eb9c17c34626c8f682 Mon Sep 17 00:00:00 2001
From: Anthony MOI <m.anthony.moi@gmail.com>
Date: Thu, 21 May 2020 16:33:52 -0400
Subject: [PATCH] Python - Also allow creating Tokenizer from_buffer

---
 bindings/python/src/tokenizer.rs        | 12 ++++++++++++
 bindings/python/tokenizers/__init__.pyi | 12 ++++++++++++
 2 files changed, 24 insertions(+)

diff --git a/bindings/python/src/tokenizer.rs b/bindings/python/src/tokenizer.rs
index 81394e99..db87ce2c 100644
--- a/bindings/python/src/tokenizer.rs
+++ b/bindings/python/src/tokenizer.rs
@@ -253,6 +253,18 @@ impl Tokenizer {
         })
     }
 
+    #[staticmethod]
+    fn from_buffer(buffer: &PyBytes) -> PyResult<Self> {
+        let tokenizer: tk::tokenizer::Tokenizer = serde_json::from_slice(buffer.as_bytes())
+            .map_err(|e| {
+                exceptions::Exception::py_err(format!(
+                    "Cannot instantiate Tokenizer from buffer: {}",
+                    e.to_string()
+                ))
+            })?;
+        Ok(Self { tokenizer })
+    }
+
     #[args(pretty = false)]
     fn to_str(&self, pretty: bool) -> PyResult<String> {
         ToPyResult(self.tokenizer.to_string(pretty)).into()
diff --git a/bindings/python/tokenizers/__init__.pyi b/bindings/python/tokenizers/__init__.pyi
index 7fa34405..dcde5063 100644
--- a/bindings/python/tokenizers/__init__.pyi
+++ b/bindings/python/tokenizers/__init__.pyi
@@ -272,6 +272,18 @@ class Tokenizer:
             path: str:
                 Path to a file containing a Tokenizer
 
+        Returns:
+            Tokenizer
+        """
+        pass
+    @staticmethod
+    def from_buffer(buffer: bytes) -> Tokenizer:
+        """ Instantiate a new Tokenizer from the given buffer
+
+        Args:
+            buffer: bytes:
+                A buffer used to instantiate a new Tokenizer
+
         Returns:
             Tokenizer
         """