Quick improvement over python bindings

bindings/python/src/lib.rs

@@ -1,4 +1,5 @@
 extern crate tokenizers as tk;
+
 use pyo3::prelude::*;
 use pyo3::wrap_pyfunction;
 
@@ -6,8 +7,7 @@ use pyo3::wrap_pyfunction;
 /// Tokenize
 fn tokenize(a: String) -> PyResult<Vec<u32>> {
     println!("Tokenize in rust");
-    tk::test();
-    Ok(vec![1, 2, 3])
+    Ok(tk::tokenize(&a))
 }
 
 #[pymodule]
@@ -20,6 +20,6 @@ fn tokenizers(py: Python, m: &PyModule) -> PyResult<()> {
 mod tests {
     #[test]
     fn it_works() {
-        assert_eq!(2 + 2, 4);
+        assert_eq!(tk::tokenize("Hey man!"), vec![1, 2, 3]);
     }
 }

tokenizers/Cargo.lock

@@ -1,6 +1,6 @@
 # This file is automatically @generated by Cargo.
 # It is not intended for manual editing.
 [[package]]
-name = "tokenizers"
+name = "tokenizers-lib"
 version = "0.1.0"
 

tokenizers/src/cli.rs

@@ -1,7 +1,8 @@
 /// This is the CLI binary for the Tokenizers project
-use tokenizers::test;
+use tokenizers;
 
 fn main() {
     println!("Hello, world!");
-    test();
+    let s = "Hey man!";
+    println!("Tokenizing {:?} gives {:?}", s, tokenizers::tokenize(&s));
 }

tokenizers/src/lib.rs

@@ -1,3 +1,3 @@
-pub fn test() {
-    println!("Test");
+pub fn tokenize(s: &str) -> Vec<u32> {
+    vec![1, 2, 3]
 }