mii/tokenizers
1
0
Fork 0
mirror of https://github.com/mii443/tokenizers.git synced 2025-08-22 16:25:30 +00:00
Code Issues Packages Projects Releases Wiki Activity

Adding a test to check that serialization of JsPreTokenizer fits

Browse Source 
PreTokenizer serialization.
This commit is contained in:
Nicolas Patry
2020-09-02 16:56:18 +02:00
committed by Anthony MOI
parent 52e527dc9f
commit b8d6c1dece
5 changed files with 154 additions and 44 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
bindings/node/native/Cargo.toml
View File

@ -19,3 +19,4 @@ neon-runtime = "0.3"
neon-serde = "0.3"
serde = { version = "1.0", features = [ "rc", "derive" ] }
tokenizers = { path = "../../../tokenizers" }
serde_json = "1.0"