Python - Also allow creating Tokenizer from_buffer

bindings/python/src/tokenizer.rs

@@ -253,6 +253,18 @@ impl Tokenizer {
         })
     }
 
+    #[staticmethod]
+    fn from_buffer(buffer: &PyBytes) -> PyResult<Self> {
+        let tokenizer: tk::tokenizer::Tokenizer = serde_json::from_slice(buffer.as_bytes())
+            .map_err(|e| {
+                exceptions::Exception::py_err(format!(
+                    "Cannot instantiate Tokenizer from buffer: {}",
+                    e.to_string()
+                ))
+            })?;
+        Ok(Self { tokenizer })
+    }
+
     #[args(pretty = false)]
     fn to_str(&self, pretty: bool) -> PyResult<String> {
         ToPyResult(self.tokenizer.to_string(pretty)).into()

bindings/python/tokenizers/__init__.pyi

@@ -272,6 +272,18 @@ class Tokenizer:
             path: str:
                 Path to a file containing a Tokenizer
 
+        Returns:
+            Tokenizer
+        """
+        pass
+    @staticmethod
+    def from_buffer(buffer: bytes) -> Tokenizer:
+        """ Instantiate a new Tokenizer from the given buffer
+
+        Args:
+            buffer: bytes:
+                A buffer used to instantiate a new Tokenizer
+
         Returns:
             Tokenizer
         """