mirror of
https://github.com/mii443/tokenizers.git
synced 2025-08-22 16:25:30 +00:00
d2c8190a0f
* Creating `normalizers.Prepend` (To be used instead of `Metaspace`). * Linting + stub. * Fixing pickling/unpickling by setting a default. * Black.
18 lines
684 B
JavaScript
18 lines
684 B
JavaScript
const native = require("./native");
|
|
|
|
module.exports = {
|
|
bertNormalizer: native.normalizers_BertNormalizer,
|
|
nfcNormalizer: native.normalizers_NFC,
|
|
nfdNormalizer: native.normalizers_NFD,
|
|
nfkcNormalizer: native.normalizers_NFKC,
|
|
nfkdNormalizer: native.normalizers_NFKD,
|
|
sequenceNormalizer: native.normalizers_Sequence,
|
|
lowercaseNormalizer: native.normalizers_Lowercase,
|
|
stripNormalizer: native.normalizers_Strip,
|
|
prependNormalizer: native.normalizers_Prepend,
|
|
stripAccentsNormalizer: native.normalizers_StripAccents,
|
|
nmtNormalizer: native.normalizers_Nmt,
|
|
precompiledNormalizer: native.normalizers_Precompiled,
|
|
replaceNormalizer: native.normalizers_Replace,
|
|
};
|