mii/tokenizers
1
0
Fork 0
mirror of https://github.com/mii443/tokenizers.git synced 2025-09-02 07:19:24 +00:00
Code Issues Packages Projects Releases Wiki Activity

Fix benchmarks

Browse Source
This commit is contained in:
Anthony MOI
2020-01-02 20:05:28 -05:00
parent 408490e6b4
commit c5359ddd47
1 changed files with 4 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
tokenizers/benches/bpe_benchmark.rs
View File

@ -58,7 +58,10 @@ fn line_to_input(line: io::Result<String>) -> EncodeInput {
} }
fn bench_gpt2(c: &mut Criterion) { fn bench_gpt2(c: &mut Criterion) {
let bpe = BPE::from_files("benches/gpt2-vocab.json", "benches/gpt2-merges.txt").unwrap(); let bpe = BPE::from_files("benches/gpt2-vocab.json", "benches/gpt2-merges.txt")
.unwrap()
.build()
.unwrap();
// Benchmarks encoding a single input from a fresh tokenizer. // Benchmarks encoding a single input from a fresh tokenizer.
c.bench_function("BPE GPT2 encode", |b| { c.bench_function("BPE GPT2 encode", |b| {