mii/tokenizers
1
0
Fork 0
mirror of https://github.com/mii443/tokenizers.git synced 2025-12-07 05:08:24 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #95 from huggingface/vocab-serialization

Browse Source 
save BPE vocab in order of ID
This commit is contained in:
MOI Anthony
2020-01-22 15:49:48 -05:00
committed by GitHub
parent 36cd67fdf5 3a9badd2e0
commit 327de00d71
3 changed files with 43 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
bindings/python/Cargo.lock generated
View File

@@ -564,6 +564,7 @@ dependencies = [
"rayon 1.3.0 (registry+https://github.com/rust-lang/crates.io-index)",
"regex 1.3.1 (registry+https://github.com/rust-lang/crates.io-index)",
"regex-syntax 0.6.12 (registry+https://github.com/rust-lang/crates.io-index)",
"serde 1.0.104 (registry+https://github.com/rust-lang/crates.io-index)",
"serde_json 1.0.44 (registry+https://github.com/rust-lang/crates.io-index)",
"unicode-normalization-alignments 0.1.12 (registry+https://github.com/rust-lang/crates.io-index)",
"unicode_categories 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)",