mii/tokenizers
1
0
Fork 0
mirror of https://github.com/mii443/tokenizers.git synced 2025-12-03 11:18:29 +00:00
Code Issues Packages Projects Releases Wiki Activity

Adding unstable_wasm feature + example to run tokenizers on wasm. (#1009)

Browse Source 
* Adding `unstable_wasm` feature + example to run `tokenizers` on wasm.

Co-Authored-By: josephrocca <1167575+josephrocca@users.noreply.github.com>
Co-Authored-By: Matthias Brunel <matthias.brunel@mithrilsecurity.io>

* Adding some serialization tests.

* Updating with comments.

Co-authored-by: josephrocca <1167575+josephrocca@users.noreply.github.com>
Co-authored-by: Matthias Brunel <matthias.brunel@mithrilsecurity.io>
This commit is contained in:
Nicolas Patry
2022-06-10 14:58:02 +02:00
committed by GitHub
parent 943b5421aa
commit adf90dcd72
30 changed files with 12883 additions and 162 deletions
Download Patch File Download Diff File Expand all files Collapse all files

81
bindings/python/Cargo.lock generated
View File

@@ -138,7 +138,7 @@ dependencies = [
"glob",
"indicatif 0.16.2",
"log",
"rand 0.8.5",
"rand",
"reqwest",
"serde",
"serde_json",
@@ -407,9 +407,9 @@ dependencies = [
[[package]]
name = "esaxx-rs"
version = "0.1.7"
version = "0.1.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6f4617b351b734e97a3dd32022a721471349aa3038d4132beee8568cdfa7e716"
checksum = "1f748b253ceca9fed5f42f8b5ceb3851e93102199bc25b64b65369f76e5c0a35"
dependencies = [
"cc",
]
@@ -554,20 +554,9 @@ dependencies = [
[[package]]
name = "getrandom"
version = "0.1.16"
version = "0.2.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8fc3cb4d91f53b50155bdcfd23f6a4c39ae1969c2ae85982b135750cccaf5fce"
dependencies = [
"cfg-if",
"libc",
"wasi 0.9.0+wasi-snapshot-preview1",
]
[[package]]
name = "getrandom"
version = "0.2.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d39cd93900197114fa1fcb7ae84ca742095eed9442088988ae74fa744e930e77"
checksum = "9be70c98951c83b8d2f8f60d7065fa6d5146873094452a1008da8c2f1e4205ad"
dependencies = [
"cfg-if",
"libc",
@@ -1288,19 +1277,6 @@ version = "0.5.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "941ba9d78d8e2f7ce474c015eea4d9c6d25b6a3327f9832ee29a4de27f91bbb8"
[[package]]
name = "rand"
version = "0.7.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6a6b1679d49b24bbfe0c803429aa1874472f50d9b363131f0e89fc356b544d03"
dependencies = [
"getrandom 0.1.16",
"libc",
"rand_chacha 0.2.2",
"rand_core 0.5.1",
"rand_hc",
]
[[package]]
name = "rand"
version = "0.8.5"
@@ -1308,18 +1284,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404"
dependencies = [
"libc",
"rand_chacha 0.3.1",
"rand_core 0.6.3",
]
[[package]]
name = "rand_chacha"
version = "0.2.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f4c8ed856279c9737206bf725bf36935d8666ead7aa69b52be55af369d193402"
dependencies = [
"ppv-lite86",
"rand_core 0.5.1",
"rand_chacha",
"rand_core",
]
[[package]]
@@ -1329,16 +1295,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88"
dependencies = [
"ppv-lite86",
"rand_core 0.6.3",
]
[[package]]
name = "rand_core"
version = "0.5.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "90bde5296fc891b0cef12a6d03ddccc162ce7b2aff54160af9338f8d40df6d19"
dependencies = [
"getrandom 0.1.16",
"rand_core",
]
[[package]]
@@ -1347,16 +1304,7 @@ version = "0.6.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d34f1408f55294453790c48b2f1ebbb1c5b4b7563eb1f418bcfcfdbb06ebb4e7"
dependencies = [
"getrandom 0.2.5",
]
[[package]]
name = "rand_hc"
version = "0.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ca3129af7b92a17112d59ad498c6f81eaf463253766b90396d39ea7a39d6613c"
dependencies = [
"rand_core 0.5.1",
"getrandom",
]
[[package]]
@@ -1416,7 +1364,7 @@ version = "0.4.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7776223e2696f1aa4c6b0170e83212f47296a00424305117d013dfe86fb0fe55"
dependencies = [
"getrandom 0.2.5",
"getrandom",
"redox_syscall",
"thiserror",
]
@@ -1759,6 +1707,7 @@ dependencies = [
"derive_builder",
"dirs",
"esaxx-rs",
"getrandom",
"indicatif 0.15.0",
"itertools 0.9.0",
"lazy_static",
@@ -1766,7 +1715,7 @@ dependencies = [
"macro_rules_attribute",
"onig",
"paste",
"rand 0.7.3",
"rand",
"rayon",
"rayon-cond",
"regex",
@@ -1971,12 +1920,6 @@ dependencies = [
"try-lock",
]
[[package]]
name = "wasi"
version = "0.9.0+wasi-snapshot-preview1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cccddf32554fecc6acb585f82a32a72e28b48f8c4c1883ddfeeeaa96f7d8e519"
[[package]]
name = "wasi"
version = "0.10.2+wasi-snapshot-preview1"