Python - Add ByteLevel PostProcessor

bindings/python/src/lib.rs

@@ -62,6 +62,7 @@ fn processors(_py: Python, m: &PyModule) -> PyResult<()> {
     m.add_class::<processors::PostProcessor>()?;
     m.add_class::<processors::BertProcessing>()?;
     m.add_class::<processors::RobertaProcessing>()?;
+    m.add_class::<processors::ByteLevel>()?;
     Ok(())
 }
 

bindings/python/src/processors.rs

@@ -42,3 +42,17 @@ impl RobertaProcessing {
         }))
     }
 }
+
+#[pyclass(extends=PostProcessor)]
+pub struct ByteLevel {}
+#[pymethods]
+impl ByteLevel {
+    #[new]
+    fn new(obj: &PyRawObject) -> PyResult<()> {
+        Ok(obj.init(PostProcessor {
+            processor: Container::Owned(Box::new(tk::processors::byte_level::ByteLevel::new(
+                false,
+            ))),
+        }))
+    }
+}

bindings/python/tokenizers/processors/__init__.py

@@ -3,3 +3,4 @@ from .. import processors
 PostProcessor = processors.PostProcessor
 BertProcessing = processors.BertProcessing
 RobertaProcessing = processors.RobertaProcessing
+ByteLevel = processors.ByteLevel

bindings/python/tokenizers/processors/__init__.pyi

@@ -62,3 +62,15 @@ class RobertaProcessing(PostProcessor):
             PostProcessor
         """
         pass
+
+class ByteLevel(PostProcessor):
+    """ ByteLevel Post processing
+
+    This post-processor takes care of fixing the offsets after the BPE Model may have
+    produced some bad offsets while merging. This happens for any unicode character that
+    get split up in many byte-level characters.
+    """
+
+    def __init(self) -> None:
+        """ Instantiate a new ByteLevel """
+        pass