From 2b72a5737f77aae947ef255abadf9fa423dc685d Mon Sep 17 00:00:00 2001
From: Anthony MOI <m.anthony.moi@gmail.com>
Date: Fri, 1 Nov 2019 13:54:17 -0400
Subject: [PATCH] Basic bin+lib setup

---
 tokenizers/Cargo.lock |  6 ++++++
 tokenizers/Cargo.toml | 15 +++++++++++++++
 tokenizers/src/cli.rs |  7 +++++++
 tokenizers/src/lib.rs |  3 +++
 4 files changed, 31 insertions(+)
 create mode 100644 tokenizers/Cargo.lock
 create mode 100644 tokenizers/Cargo.toml
 create mode 100644 tokenizers/src/cli.rs
 create mode 100644 tokenizers/src/lib.rs

diff --git a/tokenizers/Cargo.lock b/tokenizers/Cargo.lock
new file mode 100644
index 00000000..aba0ca27
--- /dev/null
+++ b/tokenizers/Cargo.lock
@@ -0,0 +1,6 @@
+# This file is automatically @generated by Cargo.
+# It is not intended for manual editing.
+[[package]]
+name = "tokenizers"
+version = "0.1.0"
+
diff --git a/tokenizers/Cargo.toml b/tokenizers/Cargo.toml
new file mode 100644
index 00000000..d5cf557e
--- /dev/null
+++ b/tokenizers/Cargo.toml
@@ -0,0 +1,15 @@
+[package]
+name = "tokenizers"
+version = "0.1.0"
+authors = ["Anthony MOI <m.anthony.moi@gmail.com>"]
+edition = "2018"
+
+[lib]
+name = "tokenizers"
+path = "src/lib.rs"
+
+[[bin]]
+name = "cli"
+path = "src/cli.rs"
+
+[dependencies]
diff --git a/tokenizers/src/cli.rs b/tokenizers/src/cli.rs
new file mode 100644
index 00000000..0cb4f40c
--- /dev/null
+++ b/tokenizers/src/cli.rs
@@ -0,0 +1,7 @@
+/// This is the CLI binary for the Tokenizers project
+use tokenizers::test;
+
+fn main() {
+    println!("Hello, world!");
+    test();
+}
diff --git a/tokenizers/src/lib.rs b/tokenizers/src/lib.rs
new file mode 100644
index 00000000..49185605
--- /dev/null
+++ b/tokenizers/src/lib.rs
@@ -0,0 +1,3 @@
+pub fn test() {
+    println!("Test");
+}