Python - Expose ByteLevel alphabet

2025-08-22 16:25:30 +00:00 · 2020-01-02 18:06:06 -05:00
parent f0f9aefd07
commit d3c3f5a700
1 changed files with 8 additions and 0 deletions
--- a/bindings/python/src/pre_tokenizers.rs
+++ b/bindings/python/src/pre_tokenizers.rs
@ -50,6 +50,14 @@ impl ByteLevel {
            ))),
        })
    }
+
+    #[staticmethod]
+    fn alphabet() -> Vec<String> {
+        tk::pre_tokenizers::byte_level::ByteLevel::alphabet()
+            .into_iter()
+            .map(|c| c.to_string())
+            .collect()
+    }
 }

 #[pyclass]