diff --git a/bindings/python/src/lib.rs b/bindings/python/src/lib.rs index 2ecd6709..796156bc 100644 --- a/bindings/python/src/lib.rs +++ b/bindings/python/src/lib.rs @@ -35,6 +35,7 @@ fn models(_py: Python, m: &PyModule) -> PyResult<()> { fn pre_tokenizers(_py: Python, m: &PyModule) -> PyResult<()> { m.add_class::()?; m.add_class::()?; + m.add_class::()?; m.add_class::()?; Ok(()) } diff --git a/bindings/python/src/pre_tokenizers.rs b/bindings/python/src/pre_tokenizers.rs index 9ba49580..e1c18b7b 100644 --- a/bindings/python/src/pre_tokenizers.rs +++ b/bindings/python/src/pre_tokenizers.rs @@ -52,6 +52,18 @@ impl ByteLevel { } } +#[pyclass] +pub struct Whitespace {} +#[pymethods] +impl Whitespace { + #[staticmethod] + fn new() -> PyResult { + Ok(PreTokenizer { + pretok: Container::Owned(Box::new(tk::pre_tokenizers::whitespace::Whitespace)), + }) + } +} + #[pyclass] pub struct BertPreTokenizer {} #[pymethods]