diff --git a/bindings/python/Cargo.lock b/bindings/python/Cargo.lock index 701531c0..78de1e4d 100644 --- a/bindings/python/Cargo.lock +++ b/bindings/python/Cargo.lock @@ -523,15 +523,15 @@ dependencies = [ [[package]] name = "tokenizers" -version = "0.1.0" +version = "0.0.1" dependencies = [ "pyo3 0.8.2 (registry+https://github.com/rust-lang/crates.io-index)", - "tokenizers-lib 0.1.0", + "tokenizers-lib 0.0.1", ] [[package]] name = "tokenizers-lib" -version = "0.1.0" +version = "0.0.1" dependencies = [ "lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)", "onig 5.0.0 (registry+https://github.com/rust-lang/crates.io-index)", diff --git a/bindings/python/Cargo.toml b/bindings/python/Cargo.toml index b5752f1c..2664c01e 100644 --- a/bindings/python/Cargo.toml +++ b/bindings/python/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "tokenizers" -version = "0.1.0" +version = "0.0.1" authors = ["Anthony MOI "] edition = "2018" diff --git a/bindings/python/README.md b/bindings/python/README.md new file mode 100644 index 00000000..7d2131c0 --- /dev/null +++ b/bindings/python/README.md @@ -0,0 +1,10 @@ +### Python Bindings + +``` +python3.7 -m venv .env +source .env/bin/activate +pip install maturin +maturin build + +python example.py +``` diff --git a/bindings/python/example.py b/bindings/python/example.py new file mode 100644 index 00000000..dded8b7a --- /dev/null +++ b/bindings/python/example.py @@ -0,0 +1,3 @@ +from tokenizers import WhitespaceTokenizer + +print(WhitespaceTokenizer.tokenize("Hey man!")) diff --git a/tokenizers/Cargo.toml b/tokenizers/Cargo.toml index 0423c6da..9f595c55 100644 --- a/tokenizers/Cargo.toml +++ b/tokenizers/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "tokenizers-lib" -version = "0.1.0" +version = "0.0.1" authors = ["Anthony MOI "] edition = "2018"