mirror of
https://github.com/mii443/tokenizers.git
synced 2025-08-23 00:35:35 +00:00
9 lines
219 B
JavaScript
9 lines
219 B
JavaScript
const tokenizers = require('..');
|
|
|
|
let bpe = tokenizers.models.BPE.from_files(
|
|
"../../data/gpt2-vocab.json",
|
|
"../../data/gpt2-merges.txt"
|
|
);
|
|
let tokenizer = new tokenizers.Tokenizer(bpe);
|
|
console.log(bpe, tokenizer);
|