mii/tokenizers
1
0
Fork 0
mirror of https://github.com/mii443/tokenizers.git synced 2025-08-22 16:25:30 +00:00
Code Issues Packages Projects Releases Wiki Activity

Quick improvement over python bindings

Browse Source
This commit is contained in:
Anthony MOI
2019-11-01 16:08:10 -04:00
parent 5d37cfde7f
commit 8448d50e6f
4 changed files with 9 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
bindings/python/src/lib.rs
View File

@ -1,4 +1,5 @@
extern crate tokenizers as tk;
use pyo3::prelude::*;
use pyo3::wrap_pyfunction;
@ -6,8 +7,7 @@ use pyo3::wrap_pyfunction;
/// Tokenize
fn tokenize(a: String) -> PyResult<Vec<u32>> {
println!("Tokenize in rust");
tk::test();
Ok(vec![1, 2, 3])
Ok(tk::tokenize(&a))
}
#[pymodule]
@ -20,6 +20,6 @@ fn tokenizers(py: Python, m: &PyModule) -> PyResult<()> {
mod tests {
#[test]
fn it_works() {
assert_eq!(2 + 2, 4);
assert_eq!(tk::tokenize("Hey man!"), vec![1, 2, 3]);
}
}