Basic bin+lib setup

tokenizers/Cargo.lock

@@ -0,0 +1,6 @@
+# This file is automatically @generated by Cargo.
+# It is not intended for manual editing.
+[[package]]
+name = "tokenizers"
+version = "0.1.0"
+

tokenizers/Cargo.toml

@@ -0,0 +1,15 @@
+[package]
+name = "tokenizers"
+version = "0.1.0"
+authors = ["Anthony MOI <m.anthony.moi@gmail.com>"]
+edition = "2018"
+
+[lib]
+name = "tokenizers"
+path = "src/lib.rs"
+
+[[bin]]
+name = "cli"
+path = "src/cli.rs"
+
+[dependencies]

tokenizers/src/cli.rs

@@ -0,0 +1,7 @@
+/// This is the CLI binary for the Tokenizers project
+use tokenizers::test;
+
+fn main() {
+    println!("Hello, world!");
+    test();
+}

tokenizers/src/lib.rs

@@ -0,0 +1,3 @@
+pub fn test() {
+    println!("Test");
+}