mii/tokenizers
1
0
Fork 0
mirror of https://github.com/mii443/tokenizers.git synced 2025-12-09 22:28:29 +00:00
Code Issues Packages Projects Releases Wiki Activity

save vocab in order of ID

Browse Source
This commit is contained in:
epwalsh
2020-01-21 13:32:13 -08:00
parent da7e629e4a
commit 3a9badd2e0
3 changed files with 43 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
bindings/python/Cargo.lock generated
View File

@@ -564,6 +564,7 @@ dependencies = [
"rayon 1.3.0 (registry+https://github.com/rust-lang/crates.io-index)",
"regex 1.3.1 (registry+https://github.com/rust-lang/crates.io-index)",
"regex-syntax 0.6.12 (registry+https://github.com/rust-lang/crates.io-index)",
"serde 1.0.104 (registry+https://github.com/rust-lang/crates.io-index)",
"serde_json 1.0.44 (registry+https://github.com/rust-lang/crates.io-index)",
"unicode-normalization-alignments 0.1.12 (registry+https://github.com/rust-lang/crates.io-index)",
"unicode_categories 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)",