mii/tokenizers
1
0
Fork 0
mirror of https://github.com/mii443/tokenizers.git synced 2025-08-22 16:25:30 +00:00
Code Issues Packages Projects Releases Wiki Activity

Node - Basic Tokenizer + BPE

Browse Source
This commit is contained in:
Anthony MOI
2020-01-09 00:04:53 -05:00
parent 13f3fbed30
commit 156d86d91e
8 changed files with 510 additions and 233 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
bindings/node/native/Cargo.toml
View File

@ -15,4 +15,4 @@ neon-build = "0.3.3"
[dependencies]
neon = "0.3.3"
tokenizers-lib = { path = "../../../tokenizers" }
tokenizers = { path = "../../../tokenizers" }