This commit is contained in:
Pierric Cistac
2020-01-13 14:48:49 -05:00
parent 6a9f7f4304
commit f836f2109b
24 changed files with 186 additions and 111 deletions

View File

@ -1,8 +1,8 @@
var addon = require('../bin-package');
var native = require('./native');
module.exports = {
byteLevelDecoder: addon.decoders_ByteLevel,
wordPieceDecoder: addon.decoders_WordPiece,
metaspaceDecoder: addon.decoders_Metaspace,
bpeDecoder: addon.decoders_BPEDecoder
byteLevelDecoder: native.decoders_ByteLevel,
wordPieceDecoder: native.decoders_WordPiece,
metaspaceDecoder: native.decoders_Metaspace,
bpeDecoder: native.decoders_BPEDecoder
};

View File

@ -1,12 +1,12 @@
var addon = require('../bin-package');
var native = require('./native');
module.exports = {
bpe: {
fromFiles: addon.models_BPE_from_files,
empty: addon.models_BPE_empty,
},
wordPiece: {
fromFiles: addon.models_WordPiece_from_files,
empty: addon.models_WordPiece_empty,
}
bpe: {
fromFiles: native.models_BPE_from_files,
empty: native.models_BPE_empty,
},
wordPiece: {
fromFiles: native.models_WordPiece_from_files,
empty: native.models_WordPiece_empty,
}
}

View File

@ -0,0 +1,2 @@
const addon = require('../../native');
module.exports = addon;

View File

@ -0,0 +1,2 @@
const native = require('../bin-package');
exports = native;

View File

@ -1,11 +1,11 @@
var addon = require('../bin-package');
var native = require('./native');
module.exports = {
bertNormalizer: addon.normalizers_BertNormalizer,
nfdNormalizer: addon.normalizers_NFD,
nfkdNormalizer: addon.normalizers_NFKD,
nfcNormalizer: addon.normalizers_NFC,
nfkcNormalizer: addon.normalizers_NFKC,
sequenceNormalizer: addon.normalizers_Sequence,
lowercaseNormalizer: addon.normalizers_Lowercase
bertNormalizer: native.normalizers_BertNormalizer,
nfdNormalizer: native.normalizers_NFD,
nfkdNormalizer: native.normalizers_NFKD,
nfcNormalizer: native.normalizers_NFC,
nfkcNormalizer: native.normalizers_NFKC,
sequenceNormalizer: native.normalizers_Sequence,
lowercaseNormalizer: native.normalizers_Lowercase
};

View File

@ -1,5 +1,5 @@
var addon = require('../bin-package');
var native = require('./native');
module.exports = {
bertProcessing: addon.processors_BertProcessing
bertProcessing: native.processors_BertProcessing
};

View File

@ -1,9 +1,9 @@
var addon = require('../bin-package');
var native = require('./native');
module.exports = {
byteLevelPreTokenizer: addon.pre_tokenizers_ByteLevel,
byteLevelAlphabet: addon.pre_tokenizers_ByteLevel_Alphabet,
whitespacePreTokenizer: addon.pre_tokenizers_Whitespace,
bertPreTokenizer: addon.pre_tokenizers_BertPreTokenizer,
metaspacePreTokenizer: addon.pre_tokenizers_Metaspace
byteLevelPreTokenizer: native.pre_tokenizers_ByteLevel,
byteLevelAlphabet: native.pre_tokenizers_ByteLevel_Alphabet,
whitespacePreTokenizer: native.pre_tokenizers_Whitespace,
bertPreTokenizer: native.pre_tokenizers_BertPreTokenizer,
metaspacePreTokenizer: native.pre_tokenizers_Metaspace
};

View File

@ -1,3 +1,5 @@
var addon = require('../bin-package');
var native = require('./native');
module.exports.Tokenizer = addon.tokenizer_Tokenizer;
module.exports = {
Tokenizer: native.tokenizer_Tokenizer
}

View File

@ -1,6 +1,6 @@
var addon = require('../bin-package');
var native = require('./native');
module.exports = {
bpeTrainer: addon.trainers_BPETrainer,
wordPieceTrainer: addon.trainers_WordPieceTrainer
bpeTrainer: native.trainers_BPETrainer,
wordPieceTrainer: native.trainers_WordPieceTrainer
};

View File

@ -1,2 +0,0 @@
*.d.ts
*.js

View File

@ -1 +0,0 @@
export * from './implementations';

View File

@ -1,7 +0,0 @@
"use strict";
function __export(m) {
for (var p in m) if (!exports.hasOwnProperty(p)) exports[p] = m[p];
}
Object.defineProperty(exports, "__esModule", { value: true });
// export * from './bindings';
__export(require("./implementations"));

View File

@ -1,2 +1,2 @@
// export * from './bindings';
export * from './implementations';
export * from './tokenizers';

View File

@ -5,7 +5,7 @@ name = "aho-corasick"
version = "0.7.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"memchr 2.2.1 (registry+https://github.com/rust-lang/crates.io-index)",
"memchr 2.3.0 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
@ -196,7 +196,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
[[package]]
name = "memchr"
version = "2.2.1"
version = "2.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
[[package]]
@ -251,7 +251,7 @@ version = "0.1.0"
dependencies = [
"neon 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)",
"neon-build 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)",
"tokenizers 0.0.13",
"tokenizers 0.6.1",
]
[[package]]
@ -275,7 +275,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
[[package]]
name = "rand"
version = "0.7.2"
version = "0.7.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"getrandom 0.1.14 (registry+https://github.com/rust-lang/crates.io-index)",
@ -338,9 +338,9 @@ version = "1.3.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"aho-corasick 0.7.6 (registry+https://github.com/rust-lang/crates.io-index)",
"memchr 2.2.1 (registry+https://github.com/rust-lang/crates.io-index)",
"memchr 2.3.0 (registry+https://github.com/rust-lang/crates.io-index)",
"regex-syntax 0.6.13 (registry+https://github.com/rust-lang/crates.io-index)",
"thread_local 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)",
"thread_local 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
@ -422,7 +422,7 @@ dependencies = [
[[package]]
name = "thread_local"
version = "1.0.0"
version = "1.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)",
@ -430,12 +430,12 @@ dependencies = [
[[package]]
name = "tokenizers"
version = "0.0.13"
version = "0.6.1"
dependencies = [
"clap 2.33.0 (registry+https://github.com/rust-lang/crates.io-index)",
"indicatif 0.13.0 (registry+https://github.com/rust-lang/crates.io-index)",
"lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)",
"rand 0.7.2 (registry+https://github.com/rust-lang/crates.io-index)",
"rand 0.7.3 (registry+https://github.com/rust-lang/crates.io-index)",
"rayon 1.3.0 (registry+https://github.com/rust-lang/crates.io-index)",
"regex 1.3.3 (registry+https://github.com/rust-lang/crates.io-index)",
"regex-syntax 0.6.13 (registry+https://github.com/rust-lang/crates.io-index)",
@ -516,7 +516,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
"checksum itoa 0.4.4 (registry+https://github.com/rust-lang/crates.io-index)" = "501266b7edd0174f8530248f87f99c88fbe60ca4ef3dd486835b8d8d53136f7f"
"checksum lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646"
"checksum libc 0.2.66 (registry+https://github.com/rust-lang/crates.io-index)" = "d515b1f41455adea1313a4a2ac8a8a477634fbae63cc6100e3aebb207ce61558"
"checksum memchr 2.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "88579771288728879b57485cc7d6b07d648c9f0141eb955f8ab7f9d45394468e"
"checksum memchr 2.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "3197e20c7edb283f87c071ddfc7a2cca8f8e0b888c242959846a6fce03c72223"
"checksum memoffset 0.5.3 (registry+https://github.com/rust-lang/crates.io-index)" = "75189eb85871ea5c2e2c15abbdd541185f63b408415e5051f5cac122d8c774b9"
"checksum neon 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)" = "53b85accbbd250627f899a6fc1f220bbb4c8c2ff6dc71830dc6b752b39c2eb97"
"checksum neon-build 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)" = "ae406bf1065c4399e69d328a3bd8d4f088f2a205dc3881bf68c0ac775bfef337"
@ -525,7 +525,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
"checksum num_cpus 1.11.1 (registry+https://github.com/rust-lang/crates.io-index)" = "76dac5ed2a876980778b8b85f75a71b6cbf0db0b1232ee12f826bccb00d09d72"
"checksum number_prefix 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "17b02fc0ff9a9e4b35b3342880f48e896ebf69f2967921fe8646bf5b7125956a"
"checksum ppv-lite86 0.2.6 (registry+https://github.com/rust-lang/crates.io-index)" = "74490b50b9fbe561ac330df47c08f3f33073d2d00c150f719147d7c54522fa1b"
"checksum rand 0.7.2 (registry+https://github.com/rust-lang/crates.io-index)" = "3ae1b169243eaf61759b8475a998f0a385e42042370f3a7dbaf35246eacc8412"
"checksum rand 0.7.3 (registry+https://github.com/rust-lang/crates.io-index)" = "6a6b1679d49b24bbfe0c803429aa1874472f50d9b363131f0e89fc356b544d03"
"checksum rand_chacha 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "03a2a90da8c7523f554344f921aa97283eadf6ac484a6d2a7d0212fa7f8d6853"
"checksum rand_core 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)" = "90bde5296fc891b0cef12a6d03ddccc162ce7b2aff54160af9338f8d40df6d19"
"checksum rand_hc 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ca3129af7b92a17112d59ad498c6f81eaf463253766b90396d39ea7a39d6613c"
@ -544,7 +544,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
"checksum strsim 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)" = "8ea5119cdb4c55b55d432abb513a0429384878c15dde60cc77b1c99de1a95a6a"
"checksum termios 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)" = "72b620c5ea021d75a735c943269bb07d30c9b77d6ac6b236bc8b5c496ef05625"
"checksum textwrap 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)" = "d326610f408c7a4eb6f51c37c330e496b08506c9457c9d34287ecc38809fb060"
"checksum thread_local 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "88ddf1ad580c7e3d1efff877d972bcc93f995556b9087a5a259630985c88ceab"
"checksum thread_local 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)" = "d40c6d1b69745a6ec6fb1ca717914848da4b44ae29d9b3080cbee91d72a69b14"
"checksum unicode-normalization-alignments 0.1.12 (registry+https://github.com/rust-lang/crates.io-index)" = "43f613e4fa046e69818dd287fdc4bc78175ff20331479dab6e1b0f98d57062de"
"checksum unicode-width 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)" = "caaa9d531767d1ff2150b9332433f32a24622147e5ebb1f26409d5da67afd479"
"checksum unicode_categories 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "39ec24b3121d976906ece63c9daad25b85969647682eee313cb5779fdd69e14e"

View File

@ -18,6 +18,7 @@
"version": "4.1.0",
"resolved": "https://registry.npmjs.org/ansi-escape-sequences/-/ansi-escape-sequences-4.1.0.tgz",
"integrity": "sha512-dzW9kHxH011uBsidTXd14JXgzye/YLb2LzeKZ4bsgl/Knwx8AtbSFkkGxagdNOoh0DlqHCmfiEjWKBaqjOanVw==",
"dev": true,
"requires": {
"array-back": "^3.0.1"
},
@ -25,14 +26,16 @@
"array-back": {
"version": "3.1.0",
"resolved": "https://registry.npmjs.org/array-back/-/array-back-3.1.0.tgz",
"integrity": "sha512-TkuxA4UCOvxuDK6NZYXCalszEzj+TLszyASooky+i742l9TqsOdYCMJJupxRic61hwquNtppB3hgcuq9SVSH1Q=="
"integrity": "sha512-TkuxA4UCOvxuDK6NZYXCalszEzj+TLszyASooky+i742l9TqsOdYCMJJupxRic61hwquNtppB3hgcuq9SVSH1Q==",
"dev": true
}
}
},
"ansi-escapes": {
"version": "3.2.0",
"resolved": "https://registry.npmjs.org/ansi-escapes/-/ansi-escapes-3.2.0.tgz",
"integrity": "sha512-cBhpre4ma+U0T1oM5fXg7Dy1Jw7zzwv7lt/GoCpr+hDQJoYnKVPLL4dCvSEFMmQurOQvSrwT7SL/DAlhBI97RQ=="
"integrity": "sha512-cBhpre4ma+U0T1oM5fXg7Dy1Jw7zzwv7lt/GoCpr+hDQJoYnKVPLL4dCvSEFMmQurOQvSrwT7SL/DAlhBI97RQ==",
"dev": true
},
"ansi-regex": {
"version": "3.0.0",
@ -43,6 +46,7 @@
"version": "3.2.1",
"resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-3.2.1.tgz",
"integrity": "sha512-VT0ZI6kZRdTh8YyJw3SMbYm/u+NqfsAxEpWO0Pf9sq8/e94WxxOpPKx9FR1FlyCtOVDNOQ+8ntlqFxiRc+r5qA==",
"dev": true,
"requires": {
"color-convert": "^1.9.0"
}
@ -65,6 +69,7 @@
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/array-back/-/array-back-2.0.0.tgz",
"integrity": "sha512-eJv4pLLufP3g5kcZry0j6WXpIbzYw9GUB4mVJZno9wfwiBxbizTnHCw3VJb07cBihbFX48Y7oSrW9y+gt4glyw==",
"dev": true,
"requires": {
"typical": "^2.6.1"
}
@ -86,12 +91,14 @@
"builtins": {
"version": "1.0.3",
"resolved": "https://registry.npmjs.org/builtins/-/builtins-1.0.3.tgz",
"integrity": "sha1-y5T662HIaWRR2zZTThQi+U8K7og="
"integrity": "sha1-y5T662HIaWRR2zZTThQi+U8K7og=",
"dev": true
},
"chalk": {
"version": "2.1.0",
"resolved": "https://registry.npmjs.org/chalk/-/chalk-2.1.0.tgz",
"integrity": "sha512-LUHGS/dge4ujbXMJrnihYMcL4AoOweGnw9Tp3kQuqy1Kx5c1qKjqvMJZ6nVJPMWJtKCTN72ZogH3oeSO9g9rXQ==",
"dev": true,
"requires": {
"ansi-styles": "^3.1.0",
"escape-string-regexp": "^1.0.5",
@ -101,7 +108,8 @@
"chardet": {
"version": "0.4.2",
"resolved": "https://registry.npmjs.org/chardet/-/chardet-0.4.2.tgz",
"integrity": "sha1-tUc7M9yXxCTl2Y3IfVXU2KKci/I="
"integrity": "sha1-tUc7M9yXxCTl2Y3IfVXU2KKci/I=",
"dev": true
},
"chownr": {
"version": "1.1.3",
@ -112,6 +120,7 @@
"version": "2.1.0",
"resolved": "https://registry.npmjs.org/cli-cursor/-/cli-cursor-2.1.0.tgz",
"integrity": "sha1-s12sN2R5+sw+lHR9QdDQ9SOP/LU=",
"dev": true,
"requires": {
"restore-cursor": "^2.0.0"
}
@ -119,7 +128,8 @@
"cli-width": {
"version": "2.2.0",
"resolved": "https://registry.npmjs.org/cli-width/-/cli-width-2.2.0.tgz",
"integrity": "sha1-/xnt6Kml5XkyQUewwR8PvLq+1jk="
"integrity": "sha1-/xnt6Kml5XkyQUewwR8PvLq+1jk=",
"dev": true
},
"code-point-at": {
"version": "1.1.0",
@ -130,6 +140,7 @@
"version": "1.9.3",
"resolved": "https://registry.npmjs.org/color-convert/-/color-convert-1.9.3.tgz",
"integrity": "sha512-QfAUtd+vFdAtFQcC8CCyYt1fYWxSqAiK2cSD6zDB8N3cpsEBAvRxp9zOGg6G/SHHJYAT88/az/IuDGALsNVbGg==",
"dev": true,
"requires": {
"color-name": "1.1.3"
}
@ -137,12 +148,14 @@
"color-name": {
"version": "1.1.3",
"resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.3.tgz",
"integrity": "sha1-p9BVi9icQveV3UIyj3QIMcpTvCU="
"integrity": "sha1-p9BVi9icQveV3UIyj3QIMcpTvCU=",
"dev": true
},
"command-line-args": {
"version": "4.0.7",
"resolved": "https://registry.npmjs.org/command-line-args/-/command-line-args-4.0.7.tgz",
"integrity": "sha512-aUdPvQRAyBvQd2n7jXcsMDz68ckBJELXNzBybCHOibUWEg0mWTnaYCSRU8h9R+aNRSvDihJtssSRCiDRpLaezA==",
"dev": true,
"requires": {
"array-back": "^2.0.0",
"find-replace": "^1.0.3",
@ -153,6 +166,7 @@
"version": "2.0.1",
"resolved": "https://registry.npmjs.org/command-line-commands/-/command-line-commands-2.0.1.tgz",
"integrity": "sha512-m8c2p1DrNd2ruIAggxd/y6DgygQayf6r8RHwchhXryaLF8I6koYjoYroVP+emeROE9DXN5b9sP1Gh+WtvTTdtQ==",
"dev": true,
"requires": {
"array-back": "^2.0.0"
}
@ -161,6 +175,7 @@
"version": "4.1.0",
"resolved": "https://registry.npmjs.org/command-line-usage/-/command-line-usage-4.1.0.tgz",
"integrity": "sha512-MxS8Ad995KpdAC0Jopo/ovGIroV/m0KHwzKfXxKag6FHOkGsH8/lv5yjgablcRxCJJC0oJeUMuO/gmaq+Wq46g==",
"dev": true,
"requires": {
"ansi-escape-sequences": "^4.0.0",
"array-back": "^2.0.0",
@ -172,6 +187,7 @@
"version": "2.20.3",
"resolved": "https://registry.npmjs.org/commander/-/commander-2.20.3.tgz",
"integrity": "sha512-GpVkmM8vF2vQUkj2LvZmD35JxeJOLCwJ9cUkugyk2nuhbv3+mJvpLYYt+0+USMxE+oj+ey/lJEnhZw75x/OMcQ==",
"dev": true,
"optional": true
},
"concat-map": {
@ -215,12 +231,14 @@
"escape-string-regexp": {
"version": "1.0.5",
"resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz",
"integrity": "sha1-G2HAViGQqN/2rjuyzwIAyhMLhtQ="
"integrity": "sha1-G2HAViGQqN/2rjuyzwIAyhMLhtQ=",
"dev": true
},
"external-editor": {
"version": "2.2.0",
"resolved": "https://registry.npmjs.org/external-editor/-/external-editor-2.2.0.tgz",
"integrity": "sha512-bSn6gvGxKt+b7+6TKEv1ZycHleA7aHhRHyAqJyp5pbUFuYYNIzpZnQDk7AsYckyWdEnTeAnay0aCy2aV6iTk9A==",
"dev": true,
"requires": {
"chardet": "^0.4.0",
"iconv-lite": "^0.4.17",
@ -231,6 +249,7 @@
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/figures/-/figures-2.0.0.tgz",
"integrity": "sha1-OrGi0qYsi/tDGgyUy3l6L84nyWI=",
"dev": true,
"requires": {
"escape-string-regexp": "^1.0.5"
}
@ -239,6 +258,7 @@
"version": "1.0.3",
"resolved": "https://registry.npmjs.org/find-replace/-/find-replace-1.0.3.tgz",
"integrity": "sha1-uI5zZNLZyVlVnziMZmcNYTBEH6A=",
"dev": true,
"requires": {
"array-back": "^1.0.4",
"test-value": "^2.1.0"
@ -248,6 +268,7 @@
"version": "1.0.4",
"resolved": "https://registry.npmjs.org/array-back/-/array-back-1.0.4.tgz",
"integrity": "sha1-ZEun8JX3/898Q7Xw3DnTwfA8Bjs=",
"dev": true,
"requires": {
"typical": "^2.6.0"
}
@ -319,6 +340,7 @@
"version": "0.0.7",
"resolved": "https://registry.npmjs.org/git-config/-/git-config-0.0.7.tgz",
"integrity": "sha1-qcij7wendsPXImE1bYtye2IgKyg=",
"dev": true,
"requires": {
"iniparser": "~1.0.5"
}
@ -337,9 +359,10 @@
}
},
"handlebars": {
"version": "4.6.0",
"resolved": "https://registry.npmjs.org/handlebars/-/handlebars-4.6.0.tgz",
"integrity": "sha512-i1ZUP7Qp2JdkMaFon2a+b0m5geE8Z4ZTLaGkgrObkEd+OkUKyRbRWw4KxuFCoHfdETSY1yf9/574eVoNSiK7pw==",
"version": "4.7.1",
"resolved": "https://registry.npmjs.org/handlebars/-/handlebars-4.7.1.tgz",
"integrity": "sha512-2dd6soo60cwKNJ90VewNLIzdZPR/E2YhszOTgHpN9V0YuwZk7x33/iZoIBnASwDFVHMY7iJ6NPL8d9f/DWYCTA==",
"dev": true,
"requires": {
"neo-async": "^2.6.0",
"optimist": "^0.6.1",
@ -350,7 +373,8 @@
"has-flag": {
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/has-flag/-/has-flag-2.0.0.tgz",
"integrity": "sha1-6CB68cx7MNRGzHC3NLXovhj4jVE="
"integrity": "sha1-6CB68cx7MNRGzHC3NLXovhj4jVE=",
"dev": true
},
"has-unicode": {
"version": "2.0.1",
@ -395,12 +419,14 @@
"iniparser": {
"version": "1.0.5",
"resolved": "https://registry.npmjs.org/iniparser/-/iniparser-1.0.5.tgz",
"integrity": "sha1-g21r7+bfv87gvM8c+fKsxwJ/eD0="
"integrity": "sha1-g21r7+bfv87gvM8c+fKsxwJ/eD0=",
"dev": true
},
"inquirer": {
"version": "3.3.0",
"resolved": "https://registry.npmjs.org/inquirer/-/inquirer-3.3.0.tgz",
"integrity": "sha512-h+xtnyk4EwKvFWHrUYsWErEVR+igKtLdchu+o0Z1RL7VU/jVMFbYir2bp6bAj8efFNxWqHX0dIss6fJQ+/+qeQ==",
"dev": true,
"requires": {
"ansi-escapes": "^3.0.0",
"chalk": "^2.0.0",
@ -432,7 +458,8 @@
"is-promise": {
"version": "2.1.0",
"resolved": "https://registry.npmjs.org/is-promise/-/is-promise-2.1.0.tgz",
"integrity": "sha1-eaKp7OfwlugPNtKy87wWwf9L8/o="
"integrity": "sha1-eaKp7OfwlugPNtKy87wWwf9L8/o=",
"dev": true
},
"isarray": {
"version": "1.0.0",
@ -442,17 +469,20 @@
"lodash": {
"version": "4.17.15",
"resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.15.tgz",
"integrity": "sha512-8xOcRHvCjnocdS5cpwXQXVzmmh5e5+saE2QGoeQmbKmRS6J3VQppPOIt0MnmE+4xlZoumy0GPG0D0MVIQbNA1A=="
"integrity": "sha512-8xOcRHvCjnocdS5cpwXQXVzmmh5e5+saE2QGoeQmbKmRS6J3VQppPOIt0MnmE+4xlZoumy0GPG0D0MVIQbNA1A==",
"dev": true
},
"lodash.padend": {
"version": "4.6.1",
"resolved": "https://registry.npmjs.org/lodash.padend/-/lodash.padend-4.6.1.tgz",
"integrity": "sha1-U8y6BH0G4VjTEfRdpiX05J5vFm4="
"integrity": "sha1-U8y6BH0G4VjTEfRdpiX05J5vFm4=",
"dev": true
},
"mimic-fn": {
"version": "1.2.0",
"resolved": "https://registry.npmjs.org/mimic-fn/-/mimic-fn-1.2.0.tgz",
"integrity": "sha512-jf84uxzwiuiIVKiOLpfYk7N46TSy8ubTonmneY9vrpHNAnp0QBt2BxWV9dO3/j+BoVAb+a5G6YDPW3M5HOdMWQ=="
"integrity": "sha512-jf84uxzwiuiIVKiOLpfYk7N46TSy8ubTonmneY9vrpHNAnp0QBt2BxWV9dO3/j+BoVAb+a5G6YDPW3M5HOdMWQ==",
"dev": true
},
"minimatch": {
"version": "3.0.4",
@ -465,7 +495,8 @@
"minimist": {
"version": "0.0.10",
"resolved": "https://registry.npmjs.org/minimist/-/minimist-0.0.10.tgz",
"integrity": "sha1-3j+YVD2/lggr5IrRoMfNqDYwHc8="
"integrity": "sha1-3j+YVD2/lggr5IrRoMfNqDYwHc8=",
"dev": true
},
"minipass": {
"version": "2.9.0",
@ -507,7 +538,8 @@
"mute-stream": {
"version": "0.0.7",
"resolved": "https://registry.npmjs.org/mute-stream/-/mute-stream-0.0.7.tgz",
"integrity": "sha1-MHXOk7whuPq0PhvE2n6BFe0ee6s="
"integrity": "sha1-MHXOk7whuPq0PhvE2n6BFe0ee6s=",
"dev": true
},
"needle": {
"version": "2.4.0",
@ -522,12 +554,14 @@
"neo-async": {
"version": "2.6.1",
"resolved": "https://registry.npmjs.org/neo-async/-/neo-async-2.6.1.tgz",
"integrity": "sha512-iyam8fBuCUpWeKPGpaNMetEocMt364qkCsfL9JuhjXX6dRnguRVOfk2GZaDpPjcOKiiXCPINZC1GczQ7iTq3Zw=="
"integrity": "sha512-iyam8fBuCUpWeKPGpaNMetEocMt364qkCsfL9JuhjXX6dRnguRVOfk2GZaDpPjcOKiiXCPINZC1GczQ7iTq3Zw==",
"dev": true
},
"neon-cli": {
"version": "0.3.3",
"resolved": "https://registry.npmjs.org/neon-cli/-/neon-cli-0.3.3.tgz",
"integrity": "sha512-P2kc9ZDp0fXfmHDx3YcQwX8gvZew8WroXG3YY6N2LZoYZNrnGUWIOYGQcck52Jmry+ka02xDtw+lCHpOuGoxRA==",
"dev": true,
"requires": {
"chalk": "~2.1.0",
"command-line-args": "^4.0.2",
@ -628,6 +662,7 @@
"version": "2.0.1",
"resolved": "https://registry.npmjs.org/onetime/-/onetime-2.0.1.tgz",
"integrity": "sha1-BnQoIw/WdEOyeUsiu6UotoZ5YtQ=",
"dev": true,
"requires": {
"mimic-fn": "^1.0.0"
}
@ -636,6 +671,7 @@
"version": "0.6.1",
"resolved": "https://registry.npmjs.org/optimist/-/optimist-0.6.1.tgz",
"integrity": "sha1-2j6nRob6IaGaERwybpDrFaAZZoY=",
"dev": true,
"requires": {
"minimist": "~0.0.1",
"wordwrap": "~0.0.2"
@ -680,6 +716,7 @@
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/quickly-copy-file/-/quickly-copy-file-1.0.0.tgz",
"integrity": "sha1-n4/wZiMFEO50IrASFHKwk6hpCFk=",
"dev": true,
"requires": {
"mkdirp": "~0.5.0"
}
@ -728,7 +765,8 @@
"reduce-flatten": {
"version": "1.0.1",
"resolved": "https://registry.npmjs.org/reduce-flatten/-/reduce-flatten-1.0.1.tgz",
"integrity": "sha1-JYx479FT3fk8tWEjf2EYTzaW4yc="
"integrity": "sha1-JYx479FT3fk8tWEjf2EYTzaW4yc=",
"dev": true
},
"resolve": {
"version": "1.14.2",
@ -743,6 +781,7 @@
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/restore-cursor/-/restore-cursor-2.0.0.tgz",
"integrity": "sha1-n37ih/gv0ybU/RYpI9YhKe7g368=",
"dev": true,
"requires": {
"onetime": "^2.0.0",
"signal-exit": "^3.0.2"
@ -759,12 +798,14 @@
"rsvp": {
"version": "4.8.5",
"resolved": "https://registry.npmjs.org/rsvp/-/rsvp-4.8.5.tgz",
"integrity": "sha512-nfMOlASu9OnRJo1mbEk2cz0D56a1MBNrJ7orjRZQG10XDyuvwksKbuXNp6qa+kbn839HwjwhBzhFmdsaEAfauA=="
"integrity": "sha512-nfMOlASu9OnRJo1mbEk2cz0D56a1MBNrJ7orjRZQG10XDyuvwksKbuXNp6qa+kbn839HwjwhBzhFmdsaEAfauA==",
"dev": true
},
"run-async": {
"version": "2.3.0",
"resolved": "https://registry.npmjs.org/run-async/-/run-async-2.3.0.tgz",
"integrity": "sha1-A3GrSuC91yDUFm19/aZP96RFpsA=",
"dev": true,
"requires": {
"is-promise": "^2.1.0"
}
@ -772,12 +813,14 @@
"rx-lite": {
"version": "4.0.8",
"resolved": "https://registry.npmjs.org/rx-lite/-/rx-lite-4.0.8.tgz",
"integrity": "sha1-Cx4Rr4vESDbwSmQH6S2kJGe3lEQ="
"integrity": "sha1-Cx4Rr4vESDbwSmQH6S2kJGe3lEQ=",
"dev": true
},
"rx-lite-aggregates": {
"version": "4.0.8",
"resolved": "https://registry.npmjs.org/rx-lite-aggregates/-/rx-lite-aggregates-4.0.8.tgz",
"integrity": "sha1-dTuHqJoRyVRnxKwWJsTvxOBcZ74=",
"dev": true,
"requires": {
"rx-lite": "*"
}
@ -826,12 +869,14 @@
"source-map": {
"version": "0.6.1",
"resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz",
"integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g=="
"integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==",
"dev": true
},
"spdx-correct": {
"version": "3.1.0",
"resolved": "https://registry.npmjs.org/spdx-correct/-/spdx-correct-3.1.0.tgz",
"integrity": "sha512-lr2EZCctC2BNR7j7WzJ2FpDznxky1sjfxvvYEyzxNyb6lZXHODmEoJeFu4JupYlkfha1KZpJyoqiJ7pgA1qq8Q==",
"dev": true,
"requires": {
"spdx-expression-parse": "^3.0.0",
"spdx-license-ids": "^3.0.0"
@ -840,12 +885,14 @@
"spdx-exceptions": {
"version": "2.2.0",
"resolved": "https://registry.npmjs.org/spdx-exceptions/-/spdx-exceptions-2.2.0.tgz",
"integrity": "sha512-2XQACfElKi9SlVb1CYadKDXvoajPgBVPn/gOQLrTvHdElaVhr7ZEbqJaRnJLVNeaI4cMEAgVCeBMKF6MWRDCRA=="
"integrity": "sha512-2XQACfElKi9SlVb1CYadKDXvoajPgBVPn/gOQLrTvHdElaVhr7ZEbqJaRnJLVNeaI4cMEAgVCeBMKF6MWRDCRA==",
"dev": true
},
"spdx-expression-parse": {
"version": "3.0.0",
"resolved": "https://registry.npmjs.org/spdx-expression-parse/-/spdx-expression-parse-3.0.0.tgz",
"integrity": "sha512-Yg6D3XpRD4kkOmTpdgbUiEJFKghJH03fiC1OPll5h/0sO6neh2jqRDVHOQ4o/LMea0tgCkbMgea5ip/e+MkWyg==",
"dev": true,
"requires": {
"spdx-exceptions": "^2.1.0",
"spdx-license-ids": "^3.0.0"
@ -854,7 +901,8 @@
"spdx-license-ids": {
"version": "3.0.5",
"resolved": "https://registry.npmjs.org/spdx-license-ids/-/spdx-license-ids-3.0.5.tgz",
"integrity": "sha512-J+FWzZoynJEXGphVIS+XEh3kFSjZX/1i9gFBaWQcB+/tmpe2qUsSBABpcxqxnAxFdiUFEgAX1bjYGQvIZmoz9Q=="
"integrity": "sha512-J+FWzZoynJEXGphVIS+XEh3kFSjZX/1i9gFBaWQcB+/tmpe2qUsSBABpcxqxnAxFdiUFEgAX1bjYGQvIZmoz9Q==",
"dev": true
},
"string-width": {
"version": "2.1.1",
@ -890,6 +938,7 @@
"version": "4.5.0",
"resolved": "https://registry.npmjs.org/supports-color/-/supports-color-4.5.0.tgz",
"integrity": "sha1-vnoN5ITexcXN34s9WRJQRJEvY1s=",
"dev": true,
"requires": {
"has-flag": "^2.0.0"
}
@ -898,6 +947,7 @@
"version": "0.4.5",
"resolved": "https://registry.npmjs.org/table-layout/-/table-layout-0.4.5.tgz",
"integrity": "sha512-zTvf0mcggrGeTe/2jJ6ECkJHAQPIYEwDoqsiqBjI24mvRmQbInK5jq33fyypaCBxX08hMkfmdOqj6haT33EqWw==",
"dev": true,
"requires": {
"array-back": "^2.0.0",
"deep-extend": "~0.6.0",
@ -924,6 +974,7 @@
"version": "2.1.0",
"resolved": "https://registry.npmjs.org/test-value/-/test-value-2.1.0.tgz",
"integrity": "sha1-Edpv9nDzRxpztiXKTz/c97t0gpE=",
"dev": true,
"requires": {
"array-back": "^1.0.3",
"typical": "^2.6.0"
@ -933,6 +984,7 @@
"version": "1.0.4",
"resolved": "https://registry.npmjs.org/array-back/-/array-back-1.0.4.tgz",
"integrity": "sha1-ZEun8JX3/898Q7Xw3DnTwfA8Bjs=",
"dev": true,
"requires": {
"typical": "^2.6.0"
}
@ -942,12 +994,14 @@
"through": {
"version": "2.3.8",
"resolved": "https://registry.npmjs.org/through/-/through-2.3.8.tgz",
"integrity": "sha1-DdTJ/6q8NXlgsbckEV1+Doai4fU="
"integrity": "sha1-DdTJ/6q8NXlgsbckEV1+Doai4fU=",
"dev": true
},
"tmp": {
"version": "0.0.33",
"resolved": "https://registry.npmjs.org/tmp/-/tmp-0.0.33.tgz",
"integrity": "sha512-jRCJlojKnZ3addtTOjdIqoRuPEKBvNXcGYqzO6zWZX8KfKEpnGY5jfggJQ3EjKuu8D4bJRr0y+cYJFmYbImXGw==",
"dev": true,
"requires": {
"os-tmpdir": "~1.0.2"
}
@ -955,12 +1009,14 @@
"toml": {
"version": "2.3.6",
"resolved": "https://registry.npmjs.org/toml/-/toml-2.3.6.tgz",
"integrity": "sha512-gVweAectJU3ebq//Ferr2JUY4WKSDe5N+z0FvjDncLGyHmIDoxgY/2Ie4qfEIDm4IS7OA6Rmdm7pdEEdMcV/xQ=="
"integrity": "sha512-gVweAectJU3ebq//Ferr2JUY4WKSDe5N+z0FvjDncLGyHmIDoxgY/2Ie4qfEIDm4IS7OA6Rmdm7pdEEdMcV/xQ==",
"dev": true
},
"ts-typed-json": {
"version": "0.2.2",
"resolved": "https://registry.npmjs.org/ts-typed-json/-/ts-typed-json-0.2.2.tgz",
"integrity": "sha1-UxhL7ok+RZkbc8jEY6OLWeJ81H4=",
"dev": true,
"requires": {
"rsvp": "^3.5.0"
},
@ -968,7 +1024,8 @@
"rsvp": {
"version": "3.6.2",
"resolved": "https://registry.npmjs.org/rsvp/-/rsvp-3.6.2.tgz",
"integrity": "sha512-OfWGQTb9vnwRjwtA2QwpG2ICclHC3pgXZO5xt8H2EfgDquO0qVdSb5T88L4qJVAEugbS56pAuV4XZM58UX8ulw=="
"integrity": "sha512-OfWGQTb9vnwRjwtA2QwpG2ICclHC3pgXZO5xt8H2EfgDquO0qVdSb5T88L4qJVAEugbS56pAuV4XZM58UX8ulw==",
"dev": true
}
}
},
@ -981,12 +1038,14 @@
"typical": {
"version": "2.6.1",
"resolved": "https://registry.npmjs.org/typical/-/typical-2.6.1.tgz",
"integrity": "sha1-XAgOXWYcu+OCWdLnCjxyU+hziB0="
"integrity": "sha1-XAgOXWYcu+OCWdLnCjxyU+hziB0=",
"dev": true
},
"uglify-js": {
"version": "3.7.4",
"resolved": "https://registry.npmjs.org/uglify-js/-/uglify-js-3.7.4.tgz",
"integrity": "sha512-tinYWE8X1QfCHxS1lBS8yiDekyhSXOO6R66yNOCdUJeojxxw+PX2BHAz/BWyW7PQ7pkiWVxJfIEbiDxyLWvUGg==",
"version": "3.7.5",
"resolved": "https://registry.npmjs.org/uglify-js/-/uglify-js-3.7.5.tgz",
"integrity": "sha512-GFZ3EXRptKGvb/C1Sq6nO1iI7AGcjyqmIyOw0DrD0675e+NNbGO72xmMM2iEBdFbxaTLo70NbjM/Wy54uZIlsg==",
"dev": true,
"optional": true,
"requires": {
"commander": "~2.20.3",
@ -1002,6 +1061,7 @@
"version": "3.0.4",
"resolved": "https://registry.npmjs.org/validate-npm-package-license/-/validate-npm-package-license-3.0.4.tgz",
"integrity": "sha512-DpKm2Ui/xN7/HQKCtpZxoRWBhZ9Z0kqtygG8XCgNQ8ZlDnxuQmWhj566j8fN4Cu3/JmbhsDo7fcAJq4s9h27Ew==",
"dev": true,
"requires": {
"spdx-correct": "^3.0.0",
"spdx-expression-parse": "^3.0.0"
@ -1011,6 +1071,7 @@
"version": "3.0.0",
"resolved": "https://registry.npmjs.org/validate-npm-package-name/-/validate-npm-package-name-3.0.0.tgz",
"integrity": "sha1-X6kS2B630MdK/BQN5zF/DKffQ34=",
"dev": true,
"requires": {
"builtins": "^1.0.3"
}
@ -1026,12 +1087,14 @@
"wordwrap": {
"version": "0.0.3",
"resolved": "https://registry.npmjs.org/wordwrap/-/wordwrap-0.0.3.tgz",
"integrity": "sha1-o9XabNXAvAAI03I0u68b7WMFkQc="
"integrity": "sha1-o9XabNXAvAAI03I0u68b7WMFkQc=",
"dev": true
},
"wordwrapjs": {
"version": "3.0.0",
"resolved": "https://registry.npmjs.org/wordwrapjs/-/wordwrapjs-3.0.0.tgz",
"integrity": "sha512-mO8XtqyPvykVCsrwj5MlOVWvSnCdT+C+QVbm6blradR7JExAhbkZ7hZ9A+9NUtwzSqrlUo9a67ws0EiILrvRpw==",
"dev": true,
"requires": {
"reduce-flatten": "^1.0.1",
"typical": "^2.6.1"

View File

@ -2,8 +2,8 @@
"name": "tokenizers",
"version": "0.2.2",
"description": "",
"main": "./lib/index.js",
"types": "./lib/index.d.ts",
"main": "./dist/index.js",
"types": "./dist/index.d.ts",
"repository": {
"type": "git",
"url": ".."
@ -12,10 +12,10 @@
"license": "Apache-2.0",
"dependencies": {
"@types/node": "^13.1.6",
"neon-cli": "^0.3.3",
"node-pre-gyp": "^0.14.0"
},
"devDependencies": {
"neon-cli": "^0.3.3",
"shelljs": "^0.8.3",
"typescript": "^3.7.4"
},
@ -23,7 +23,8 @@
"node": ">=12.0.0"
},
"scripts": {
"dev": "neon build",
"dev-ts": "rm -rf dist && tsc && ln -s $(pwd)/lib/bindings dist/bindings",
"dev-rs": "neon build",
"compile": "neon build --release",
"clean": "neon clean"
},

View File

@ -19,15 +19,17 @@ const fs = require("fs");
const path = require("path");
const shell = require("shelljs");
//Fail this script if any of these commands fail
// Fail this script if any of these commands fail
shell.set("-e");
//Ensure that our directory is set to the root of the repo
// Ensure that our directory is set to the root of the repo
const rootDirectory = path.dirname(process.argv[1]);
shell.cd(rootDirectory);
const shouldPublish = process.argv.slice(2).indexOf("--publish") !== -1;
//Cleanup the previous build, if it exists
shell.rm("-rf", "./dist");
const shouldPublish = process.argv.slice(2).indexOf("--publish") !== -1;
const distPath = "./dist";
// Cleanup the previous build, if it exists
shell.rm("-rf", distPath);
shell.rm("-rf", "./bin-package");
shell.rm("-rf", "./build");
@ -40,25 +42,35 @@ shell.popd();
shell.exec("npm run compile");
// shell.exec("npm test");
shell.mkdir("./dist");
shell.mkdir(distPath);
buildJsLib(distPath);
shell.cp("-r", ["package.json", "../../LICENSE"], distPath);
shell.cp("-r", ["README.md", "package.json", "lib/*", "../../LICENSE"], "./dist");
shell.rm(["./dist/implementations/.gitignore"]);
//Add a NPM install script to the package.json that we push to NPM so that when consumers pull it down it
//runs the expected node-pre-gyp step.
const npmPackageJson = require("./dist/package.json");
// Add a NPM install script to the package.json that we push to NPM so that when consumers pull it down it
// runs the expected node-pre-gyp step.
const npmPackageJson = require(`${distPath}/package.json`);
npmPackageJson.scripts.install = "node-pre-gyp install";
fs.writeFileSync("./dist/package.json", JSON.stringify(npmPackageJson, null, 2));
fs.writeFileSync(`${distPath}/package.json`, JSON.stringify(npmPackageJson, null, 2));
shell.mkdir("./bin-package");
shell.cp("./native/index.node", "./bin-package");
shell.exec("./node_modules/.bin/node-pre-gyp package");
var tgz = shell.exec("find ./build -name *.tar.gz");
shell.cp(tgz, "./bin-package/");
shell.pushd("./dist");
shell.pushd(distPath);
shell.exec(shouldPublish ? "npm publish --access public" : "echo 'Skipping publishing to npm...'");
shell.popd();
shell.echo("publish.js COMPLETE");
/**
* Takes care of building the js lib
* @param {string} path Location into which build the lib
*/
function buildJsLib(path) {
shell.exec("tsc");
shell.cp("-r", ["lib/bindings"], path);
shell.mv([`${path}/bindings/native.prod.js`], [`${path}/bindings/native.js`]);
}

View File

@ -1,4 +1,7 @@
{
"include": [
"./lib"
],
"compilerOptions": {
/* Basic Options */
// "incremental": true, /* Enable incremental compilation */
@ -12,7 +15,7 @@
// "declarationMap": true, /* Generates a sourcemap for each corresponding '.d.ts' file. */
// "sourceMap": true, /* Generates corresponding '.map' file. */
// "outFile": "./", /* Concatenate and emit output to single file. */
// "outDir": "./", /* Redirect output structure to the directory. */
"outDir": "./dist", /* Redirect output structure to the directory. */
"rootDir": "./lib", /* Specify the root directory of input files. Use to control the output directory structure with --outDir. */
// "composite": true, /* Enable project compilation */
// "tsBuildInfoFile": "./", /* Specify file to store incremental compilation information */
@ -39,7 +42,7 @@
// "noFallthroughCasesInSwitch": true, /* Report errors for fallthrough cases in switch statement. */
/* Module Resolution Options */
// "moduleResolution": "node", /* Specify module resolution strategy: 'node' (Node.js) or 'classic' (TypeScript pre-1.6). */
"moduleResolution": "node", /* Specify module resolution strategy: 'node' (Node.js) or 'classic' (TypeScript pre-1.6). */
// "baseUrl": "./", /* Base directory to resolve non-absolute module names. */
// "paths": {}, /* A series of entries which re-map imports to lookup locations relative to the 'baseUrl'. */
// "rootDirs": [], /* List of root folders whose combined content represents the structure of the project at runtime. */