/* tslint:disable */ /* eslint-disable */ /* prettier-ignore */ /* auto-generated by NAPI-RS */ const { existsSync, readFileSync } = require('fs') const { join } = require('path') const { platform, arch } = process let nativeBinding = null let localFileExisted = false let loadError = null function isMusl() { // For Node 10 if (!process.report || typeof process.report.getReport !== 'function') { try { const lddPath = require('child_process').execSync('which ldd').toString().trim() return readFileSync(lddPath, 'utf8').includes('musl') } catch (e) { return true } } else { const { glibcVersionRuntime } = process.report.getReport().header return !glibcVersionRuntime } } switch (platform) { case 'android': switch (arch) { case 'arm64': localFileExisted = existsSync(join(__dirname, 'tokenizers.android-arm64.node')) try { if (localFileExisted) { nativeBinding = require('./tokenizers.android-arm64.node') } else { nativeBinding = require('tokenizers-android-arm64') } } catch (e) { loadError = e } break case 'arm': localFileExisted = existsSync(join(__dirname, 'tokenizers.android-arm-eabi.node')) try { if (localFileExisted) { nativeBinding = require('./tokenizers.android-arm-eabi.node') } else { nativeBinding = require('tokenizers-android-arm-eabi') } } catch (e) { loadError = e } break default: throw new Error(`Unsupported architecture on Android ${arch}`) } break case 'win32': switch (arch) { case 'x64': localFileExisted = existsSync(join(__dirname, 'tokenizers.win32-x64-msvc.node')) try { if (localFileExisted) { nativeBinding = require('./tokenizers.win32-x64-msvc.node') } else { nativeBinding = require('tokenizers-win32-x64-msvc') } } catch (e) { loadError = e } break case 'ia32': localFileExisted = existsSync(join(__dirname, 'tokenizers.win32-ia32-msvc.node')) try { if (localFileExisted) { nativeBinding = require('./tokenizers.win32-ia32-msvc.node') } else { nativeBinding = require('tokenizers-win32-ia32-msvc') } } catch (e) { loadError = e } break case 'arm64': localFileExisted = existsSync(join(__dirname, 'tokenizers.win32-arm64-msvc.node')) try { if (localFileExisted) { nativeBinding = require('./tokenizers.win32-arm64-msvc.node') } else { nativeBinding = require('tokenizers-win32-arm64-msvc') } } catch (e) { loadError = e } break default: throw new Error(`Unsupported architecture on Windows: ${arch}`) } break case 'darwin': localFileExisted = existsSync(join(__dirname, 'tokenizers.darwin-universal.node')) try { if (localFileExisted) { nativeBinding = require('./tokenizers.darwin-universal.node') } else { nativeBinding = require('tokenizers-darwin-universal') } break } catch {} switch (arch) { case 'x64': localFileExisted = existsSync(join(__dirname, 'tokenizers.darwin-x64.node')) try { if (localFileExisted) { nativeBinding = require('./tokenizers.darwin-x64.node') } else { nativeBinding = require('tokenizers-darwin-x64') } } catch (e) { loadError = e } break case 'arm64': localFileExisted = existsSync(join(__dirname, 'tokenizers.darwin-arm64.node')) try { if (localFileExisted) { nativeBinding = require('./tokenizers.darwin-arm64.node') } else { nativeBinding = require('tokenizers-darwin-arm64') } } catch (e) { loadError = e } break default: throw new Error(`Unsupported architecture on macOS: ${arch}`) } break case 'freebsd': if (arch !== 'x64') { throw new Error(`Unsupported architecture on FreeBSD: ${arch}`) } localFileExisted = existsSync(join(__dirname, 'tokenizers.freebsd-x64.node')) try { if (localFileExisted) { nativeBinding = require('./tokenizers.freebsd-x64.node') } else { nativeBinding = require('tokenizers-freebsd-x64') } } catch (e) { loadError = e } break case 'linux': switch (arch) { case 'x64': if (isMusl()) { localFileExisted = existsSync(join(__dirname, 'tokenizers.linux-x64-musl.node')) try { if (localFileExisted) { nativeBinding = require('./tokenizers.linux-x64-musl.node') } else { nativeBinding = require('tokenizers-linux-x64-musl') } } catch (e) { loadError = e } } else { localFileExisted = existsSync(join(__dirname, 'tokenizers.linux-x64-gnu.node')) try { if (localFileExisted) { nativeBinding = require('./tokenizers.linux-x64-gnu.node') } else { nativeBinding = require('tokenizers-linux-x64-gnu') } } catch (e) { loadError = e } } break case 'arm64': if (isMusl()) { localFileExisted = existsSync(join(__dirname, 'tokenizers.linux-arm64-musl.node')) try { if (localFileExisted) { nativeBinding = require('./tokenizers.linux-arm64-musl.node') } else { nativeBinding = require('tokenizers-linux-arm64-musl') } } catch (e) { loadError = e } } else { localFileExisted = existsSync(join(__dirname, 'tokenizers.linux-arm64-gnu.node')) try { if (localFileExisted) { nativeBinding = require('./tokenizers.linux-arm64-gnu.node') } else { nativeBinding = require('tokenizers-linux-arm64-gnu') } } catch (e) { loadError = e } } break case 'arm': localFileExisted = existsSync(join(__dirname, 'tokenizers.linux-arm-gnueabihf.node')) try { if (localFileExisted) { nativeBinding = require('./tokenizers.linux-arm-gnueabihf.node') } else { nativeBinding = require('tokenizers-linux-arm-gnueabihf') } } catch (e) { loadError = e } break case 'riscv64': if (isMusl()) { localFileExisted = existsSync(join(__dirname, 'tokenizers.linux-riscv64-musl.node')) try { if (localFileExisted) { nativeBinding = require('./tokenizers.linux-riscv64-musl.node') } else { nativeBinding = require('tokenizers-linux-riscv64-musl') } } catch (e) { loadError = e } } else { localFileExisted = existsSync(join(__dirname, 'tokenizers.linux-riscv64-gnu.node')) try { if (localFileExisted) { nativeBinding = require('./tokenizers.linux-riscv64-gnu.node') } else { nativeBinding = require('tokenizers-linux-riscv64-gnu') } } catch (e) { loadError = e } } break case 's390x': localFileExisted = existsSync(join(__dirname, 'tokenizers.linux-s390x-gnu.node')) try { if (localFileExisted) { nativeBinding = require('./tokenizers.linux-s390x-gnu.node') } else { nativeBinding = require('tokenizers-linux-s390x-gnu') } } catch (e) { loadError = e } break default: throw new Error(`Unsupported architecture on Linux: ${arch}`) } break default: throw new Error(`Unsupported OS: ${platform}, architecture: ${arch}`) } if (!nativeBinding) { if (loadError) { throw loadError } throw new Error(`Failed to load native binding`) } const { Decoder, bpeDecoder, byteFallbackDecoder, ctcDecoder, fuseDecoder, metaspaceDecoder, replaceDecoder, sequenceDecoder, stripDecoder, wordPieceDecoder, Encoding, TruncationDirection, TruncationStrategy, Model, BPE, WordPiece, WordLevel, Unigram, Normalizer, prependNormalizer, stripAccentsNormalizer, bertNormalizer, nfdNormalizer, nfkdNormalizer, nfcNormalizer, nfkcNormalizer, stripNormalizer, sequenceNormalizer, lowercase, replace, nmt, precompiled, JsSplitDelimiterBehavior, PreTokenizer, byteLevelPreTokenizer, byteLevelAlphabet, whitespacePreTokenizer, whitespaceSplitPreTokenizer, bertPreTokenizer, metaspacePreTokenizer, splitPreTokenizer, punctuationPreTokenizer, sequencePreTokenizer, charDelimiterSplit, digitsPreTokenizer, Processor, bertProcessing, robertaProcessing, byteLevelProcessing, templateProcessing, sequenceProcessing, PaddingDirection, AddedToken, Tokenizer, Trainer, slice, mergeEncodings, } = nativeBinding module.exports.Decoder = Decoder module.exports.bpeDecoder = bpeDecoder module.exports.byteFallbackDecoder = byteFallbackDecoder module.exports.ctcDecoder = ctcDecoder module.exports.fuseDecoder = fuseDecoder module.exports.metaspaceDecoder = metaspaceDecoder module.exports.replaceDecoder = replaceDecoder module.exports.sequenceDecoder = sequenceDecoder module.exports.stripDecoder = stripDecoder module.exports.wordPieceDecoder = wordPieceDecoder module.exports.Encoding = Encoding module.exports.TruncationDirection = TruncationDirection module.exports.TruncationStrategy = TruncationStrategy module.exports.Model = Model module.exports.BPE = BPE module.exports.WordPiece = WordPiece module.exports.WordLevel = WordLevel module.exports.Unigram = Unigram module.exports.Normalizer = Normalizer module.exports.prependNormalizer = prependNormalizer module.exports.stripAccentsNormalizer = stripAccentsNormalizer module.exports.bertNormalizer = bertNormalizer module.exports.nfdNormalizer = nfdNormalizer module.exports.nfkdNormalizer = nfkdNormalizer module.exports.nfcNormalizer = nfcNormalizer module.exports.nfkcNormalizer = nfkcNormalizer module.exports.stripNormalizer = stripNormalizer module.exports.sequenceNormalizer = sequenceNormalizer module.exports.lowercase = lowercase module.exports.replace = replace module.exports.nmt = nmt module.exports.precompiled = precompiled module.exports.JsSplitDelimiterBehavior = JsSplitDelimiterBehavior module.exports.PreTokenizer = PreTokenizer module.exports.byteLevelPreTokenizer = byteLevelPreTokenizer module.exports.byteLevelAlphabet = byteLevelAlphabet module.exports.whitespacePreTokenizer = whitespacePreTokenizer module.exports.whitespaceSplitPreTokenizer = whitespaceSplitPreTokenizer module.exports.bertPreTokenizer = bertPreTokenizer module.exports.metaspacePreTokenizer = metaspacePreTokenizer module.exports.splitPreTokenizer = splitPreTokenizer module.exports.punctuationPreTokenizer = punctuationPreTokenizer module.exports.sequencePreTokenizer = sequencePreTokenizer module.exports.charDelimiterSplit = charDelimiterSplit module.exports.digitsPreTokenizer = digitsPreTokenizer module.exports.Processor = Processor module.exports.bertProcessing = bertProcessing module.exports.robertaProcessing = robertaProcessing module.exports.byteLevelProcessing = byteLevelProcessing module.exports.templateProcessing = templateProcessing module.exports.sequenceProcessing = sequenceProcessing module.exports.PaddingDirection = PaddingDirection module.exports.AddedToken = AddedToken module.exports.Tokenizer = Tokenizer module.exports.Trainer = Trainer module.exports.slice = slice module.exports.mergeEncodings = mergeEncodings