mii/tokenizers
1
0
Fork 0
mirror of https://github.com/mii443/tokenizers.git synced 2025-08-22 16:25:30 +00:00
Code Issues Packages Projects Releases Wiki Activity

Add ability to train from Iterator

Browse Source
This commit is contained in:
Anthony MOI
2020-11-12 12:58:14 -05:00
committed by Anthony MOI
parent 6e364cb685
commit e0a70f1fb2
11 changed files with 380 additions and 199 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
bindings/python/Cargo.toml
View File

@ -18,6 +18,7 @@ pyo3 = "0.12"
numpy = { git = "https://github.com/pyo3/rust-numpy/", rev = "e331befa27fede78d4662edf08fa0508db39be01" }
ndarray = "0.13"
onig = { version = "6.0", default-features = false }
crossbeam = "0.8"
[dependencies.tokenizers]
version = "*"