mii/tokenizers
1
0
Fork 0
mirror of https://github.com/mii443/tokenizers.git synced 2025-08-23 00:35:35 +00:00
Code Issues Packages Projects Releases Wiki Activity

Python - Expose ByteLevel alphabet

Browse Source
This commit is contained in:
Anthony MOI
2020-01-02 18:06:06 -05:00
parent f0f9aefd07
commit d3c3f5a700
1 changed files with 8 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
bindings/python/src/pre_tokenizers.rs
View File

@ -50,6 +50,14 @@ impl ByteLevel {
))), ))),
}) })
} }
#[staticmethod]
fn alphabet() -> Vec<String> {
tk::pre_tokenizers::byte_level::ByteLevel::alphabet()
.into_iter()
.map(|c| c.to_string())
.collect()
}
} }
#[pyclass] #[pyclass]