mirror of
https://github.com/mii443/tokenizers.git
synced 2025-08-23 00:35:35 +00:00
11 lines
241 B
JavaScript
11 lines
241 B
JavaScript
var addon = require('../native');
|
|
|
|
exports.Tokenizer = addon.tokenizer_Tokenizer;
|
|
exports.models = {
|
|
BPE: {
|
|
from_files: addon.models_create_BPE_from_files,
|
|
empty: addon.models_create_BPE_empty,
|
|
},
|
|
WordPiece: addon.models_WordPiece,
|
|
}
|