Move whitespace pre tokenizer

tokenizers/src/lib.rs

@@ -1,7 +1,8 @@
 #[macro_use]
 extern crate lazy_static;
 
-mod libs;
-
-// Re-export
-pub use libs::WhitespaceTokenizer;
+pub mod models;
+pub mod normalizers;
+pub mod pre_tokenizers;
+pub mod processors;
+pub mod tokenizer;

tokenizers/src/libs/mod.rs

@@ -1,3 +0,0 @@
-mod whitespace;
-
-pub use whitespace::WhitespaceTokenizer;

tokenizers/src/pre_tokenizers/mod.rs

@@ -0,0 +1 @@
+pub mod whitespace;