mii/tokenizers
1
0
Fork 0
mirror of https://github.com/mii443/tokenizers.git synced 2025-09-02 23:39:14 +00:00
Code Issues Packages Projects Releases Wiki Activity

Node - Basic Tokenizer + BPE

Browse Source
This commit is contained in:
Anthony MOI
2020-01-09 00:04:53 -05:00
parent 13f3fbed30
commit 156d86d91e
8 changed files with 510 additions and 233 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
bindings/node/lib/index.js
View File

@ -1,8 +1,10 @@
var addon = require('../native');
let s = "Hey man!";
if (typeof process.argv[2] == 'string') {
s = process.argv[2];
exports.Tokenizer = addon.tokenizer_Tokenizer;
exports.models = {
BPE: {
from_files: addon.models_create_BPE_from_files,
empty: addon.models_create_BPE_empty,
},
WordPiece: addon.models_WordPiece,
}
console.log(addon.WhitespaceTokenizer.tokenize(s));