diff --git a/bindings/python/src/lib.rs b/bindings/python/src/lib.rs index a69c5665..b7790daf 100644 --- a/bindings/python/src/lib.rs +++ b/bindings/python/src/lib.rs @@ -1,4 +1,5 @@ extern crate tokenizers as tk; + use pyo3::prelude::*; use pyo3::wrap_pyfunction; @@ -6,8 +7,7 @@ use pyo3::wrap_pyfunction; /// Tokenize fn tokenize(a: String) -> PyResult> { println!("Tokenize in rust"); - tk::test(); - Ok(vec![1, 2, 3]) + Ok(tk::tokenize(&a)) } #[pymodule] @@ -20,6 +20,6 @@ fn tokenizers(py: Python, m: &PyModule) -> PyResult<()> { mod tests { #[test] fn it_works() { - assert_eq!(2 + 2, 4); + assert_eq!(tk::tokenize("Hey man!"), vec![1, 2, 3]); } } diff --git a/tokenizers/Cargo.lock b/tokenizers/Cargo.lock index aba0ca27..6c128c88 100644 --- a/tokenizers/Cargo.lock +++ b/tokenizers/Cargo.lock @@ -1,6 +1,6 @@ # This file is automatically @generated by Cargo. # It is not intended for manual editing. [[package]] -name = "tokenizers" +name = "tokenizers-lib" version = "0.1.0" diff --git a/tokenizers/src/cli.rs b/tokenizers/src/cli.rs index 0cb4f40c..749af42a 100644 --- a/tokenizers/src/cli.rs +++ b/tokenizers/src/cli.rs @@ -1,7 +1,8 @@ /// This is the CLI binary for the Tokenizers project -use tokenizers::test; +use tokenizers; fn main() { println!("Hello, world!"); - test(); + let s = "Hey man!"; + println!("Tokenizing {:?} gives {:?}", s, tokenizers::tokenize(&s)); } diff --git a/tokenizers/src/lib.rs b/tokenizers/src/lib.rs index 49185605..3f0170c2 100644 --- a/tokenizers/src/lib.rs +++ b/tokenizers/src/lib.rs @@ -1,3 +1,3 @@ -pub fn test() { - println!("Test"); +pub fn tokenize(s: &str) -> Vec { + vec![1, 2, 3] }