mirror of
https://github.com/mii443/tokenizers.git
synced 2025-08-22 16:25:30 +00:00
dd399d2ad0
* start playing around * make a first version * refactor * apply make format * add python bindings * add some python binding tests * correct pre-tokenizers * update auto-generated bindings * lint python bindings * add code node * add split to docs * refactor python binding a bit * cargo fmt * clippy and fmt in node * quick updates and fixes * Oops * Update node typings * Update changelog Co-authored-by: Anthony MOI <m.anthony.moi@gmail.com>
16 lines
724 B
JavaScript
16 lines
724 B
JavaScript
const native = require("./native");
|
|
|
|
module.exports = {
|
|
byteLevelPreTokenizer: native.pre_tokenizers_ByteLevel,
|
|
byteLevelAlphabet: native.pre_tokenizers_ByteLevel_Alphabet,
|
|
whitespacePreTokenizer: native.pre_tokenizers_Whitespace,
|
|
whitespaceSplitPreTokenizer: native.pre_tokenizers_WhitespaceSplit,
|
|
bertPreTokenizer: native.pre_tokenizers_BertPreTokenizer,
|
|
metaspacePreTokenizer: native.pre_tokenizers_Metaspace,
|
|
charDelimiterSplitPreTokenizer: native.pre_tokenizers_CharDelimiterSplit,
|
|
punctuationPreTokenizer: native.pre_tokenizers_Punctuation,
|
|
sequencePreTokenizer: native.pre_tokenizers_Sequence,
|
|
digitsPreTokenizer: native.pre_tokenizers_Digits,
|
|
splitPreTokenizer: native.pre_tokenizers_Split,
|
|
};
|