From 5bf8baec68bbc6cf40b83e0e85b70b764a31015c Mon Sep 17 00:00:00 2001
From: Anthony MOI <m.anthony.moi@gmail.com>
Date: Thu, 12 Dec 2019 17:24:54 -0500
Subject: [PATCH] Prepare tokenizer module for multiple files

---
 tokenizers/src/{tokenizer.rs => tokenizer/mod.rs} | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 rename tokenizers/src/{tokenizer.rs => tokenizer/mod.rs} (100%)

diff --git a/tokenizers/src/tokenizer.rs b/tokenizers/src/tokenizer/mod.rs
similarity index 100%
rename from tokenizers/src/tokenizer.rs
rename to tokenizers/src/tokenizer/mod.rs