diff --git a/tokenizers/Cargo.lock b/tokenizers/Cargo.lock new file mode 100644 index 00000000..aba0ca27 --- /dev/null +++ b/tokenizers/Cargo.lock @@ -0,0 +1,6 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +[[package]] +name = "tokenizers" +version = "0.1.0" + diff --git a/tokenizers/Cargo.toml b/tokenizers/Cargo.toml new file mode 100644 index 00000000..d5cf557e --- /dev/null +++ b/tokenizers/Cargo.toml @@ -0,0 +1,15 @@ +[package] +name = "tokenizers" +version = "0.1.0" +authors = ["Anthony MOI "] +edition = "2018" + +[lib] +name = "tokenizers" +path = "src/lib.rs" + +[[bin]] +name = "cli" +path = "src/cli.rs" + +[dependencies] diff --git a/tokenizers/src/cli.rs b/tokenizers/src/cli.rs new file mode 100644 index 00000000..0cb4f40c --- /dev/null +++ b/tokenizers/src/cli.rs @@ -0,0 +1,7 @@ +/// This is the CLI binary for the Tokenizers project +use tokenizers::test; + +fn main() { + println!("Hello, world!"); + test(); +} diff --git a/tokenizers/src/lib.rs b/tokenizers/src/lib.rs new file mode 100644 index 00000000..49185605 --- /dev/null +++ b/tokenizers/src/lib.rs @@ -0,0 +1,3 @@ +pub fn test() { + println!("Test"); +}