update build script / actions

.github/workflows/node-release.yml

@@ -8,12 +8,40 @@ env:
 on:
   push:
     tags:
-      - tokenizers-node-v*
+      - node-v*
     branches:
       - node-workflow
 
 jobs:
-  node_pre_gyp:
+  build_all:
+    name: Check everything builds
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v1
+
+      - name: Install Rust
+        uses: actions-rs/toolchain@v1
+        with:
+          toolchain: nightly-2019-11-01
+          override: true
+
+      - name: Install Node 12.x
+        uses: actions/setup-node@v1
+        with:
+          node-version: 12.x
+
+      - name: Install npm dependencies
+        working-directory: ./bindings/node
+        run: npm ci --ignore-scripts
+
+      - name: Build all
+        working-directory: ./bindings/node
+        run: node build.js --all
+
+  rust_publish:
+    name: Build and publish rust for all environments
+    needs: build_all
     strategy:
       matrix:
         os: [windows-latest, macos-latest, ubuntu-latest]
@@ -34,13 +62,13 @@ jobs:
         with:
           node-version: ${{ matrix.node-version }}
 
-      - name: Install dependencies
+      - name: Install npm dependencies
         working-directory: ./bindings/node
         run: npm ci --ignore-scripts
 
-      - name: Run publish.js for build
+      - name: Build and package rust
         working-directory: ./bindings/node
-        run: node publish.js
+        run: node build.js --package-rust
 
       - name: Install Python
         uses: actions/setup-python@v1
@@ -53,3 +81,27 @@ jobs:
         run: |
           pip install awscli
           aws s3 sync --exact-timestamps --exclude "*" --include "*.tar.gz" --acl public-read ./bin-package "s3://tokenizers-releases/node/$(node -p -e 'require("./package.json").version')-test"
+
+  npm_publish:
+    name: Build and publish JS lib
+    needs: rust_publish
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v1
+
+      - name: Install Node 12.x
+        uses: actions/setup-node@v1
+        with:
+          registry-url: https://registry.npmjs.org
+          node-version: 12.x
+
+      - name: Install npm dependencies
+        working-directory: ./bindings/node
+        run: npm ci --ignore-scripts
+
+      - name: Build and publish on NPM
+        working-directory: ./bindings/node
+        run: node build.js --npm-publish
+        env:
+          NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }}

bindings/node/build.js

@@ -0,0 +1,121 @@
+#!/usr/bin/env node
+
+/**
+ * Inspired from https://github.com/IronCoreLabs/recrypt-node-binding
+ * ==================================
+ *
+ * This script is responsible for compiling and building the NPM release bundle for this repo. The following steps are taken:
+ *
+ * + Clean up any existing Rust builds by running `cargo clean`.
+ * + Run `cargo update` to make sure all dependencies are available.
+ * + Compile rust code into index.node file.
+ * + Run unit tests to ensure the library is in good shape for publishing.
+ * + Move all expected content into a `dist` directory.
+ * + Generate a binary distrubtion in `bin-package`.
+ * + Do a dry run of npm publishing via irish-pub or perform an actual publish step if `--publish` option is provided.
+ */
+
+const fs    = require("fs");
+const path  = require("path");
+const shell = require("shelljs");
+
+const distPath = "./dist";
+
+// Fail this script if any of these commands fail
+shell.set("-e");
+
+// Ensure that our directory is set to the root of the repo
+const rootDirectory = path.dirname(process.argv[1]);
+shell.cd(rootDirectory);
+
+const arg = process.argv.slice(2)[0];
+switch (arg) {
+  case "--all":
+    buildRust();
+    buildTs();
+    break;
+
+  case "--rust":
+    buildRust();
+    break;
+
+  case "--typescript":
+    buildTs();
+    break;
+
+  case "--package-rust":
+    buildRust();
+    packageRust();
+    break;
+
+  case "--npm-publish":
+    buildTs();
+    npmPublish();
+    break;
+
+  default:
+    shell.echo('No arg provided, doing nothing...');
+    break;
+}
+
+function buildRust() {
+  shell.echo('BUILDING RUST...');
+
+  // Cleanup the previous build, if it exists
+  shell.rm("-rf", "./bin-package");
+  shell.rm("-rf", "./build");
+  
+  // Cleanup any previous Rust builds, update deps, and compile
+  shell.exec("npm ci --ignore-scripts");
+  shell.exec("npm run clean");
+  shell.pushd("./native");
+  shell.exec("cargo update");
+  shell.popd();
+  shell.exec("npm run compile");
+
+  shell.echo('BUILDING RUST COMPLETE...');
+}
+
+function packageRust() {
+  shell.echo('PACKAGING RUST...');
+
+  shell.mkdir("./bin-package");
+  shell.cp("./native/index.node", "./bin-package");
+  shell.exec("npm run package");
+  var tgz = shell.exec("find ./build -name *.tar.gz");
+  shell.cp(tgz, "./bin-package/");
+
+  shell.echo('PACKAGING RUST COMPLETE...');
+}
+
+function buildTs() {
+  shell.echo('BUILDING TS...');
+
+  // Cleanup the previous build, if it exists
+  shell.rm("-rf", distPath);
+
+  shell.exec("npm ci --ignore-scripts");
+  shell.mkdir(distPath);
+
+  shell.exec("npx tsc");
+  shell.cp("-r", ["lib/bindings"], distPath);
+  shell.mv([`${distPath}/bindings/native.prod.js`], [`${distPath}/bindings/native.js`]);
+
+  shell.echo('BUILDING TS COMPLETE...');
+}
+
+async function npmPublish() {
+  shell.echo('PUBLISHING ON NPM...');
+
+  shell.cp("-r", ["package.json", "../../LICENSE"], distPath);
+
+  // Add a NPM install script to the package.json that we push to NPM so that when consumers pull it down it
+  // runs the expected node-pre-gyp step.
+  const npmPackageJson = require(`${distPath}/package.json`);
+  npmPackageJson.scripts.install = "node-pre-gyp install";
+  await fs.promises.writeFile(`${distPath}/package.json`, JSON.stringify(npmPackageJson, null, 2));
+
+  shell.exec("npm publish --dry-run --access public");
+
+  shell.echo('PUBLISHING ON NPM COMPLETE...');
+}

bindings/node/publish.js

@@ -1,76 +0,0 @@
-#!/usr/bin/env node
-
-/**
- * Inspired from https://github.com/IronCoreLabs/recrypt-node-binding
- * ==================================
- *
- * This script is responsible for compiling and building the NPM release bundle for this repo. The following steps are taken:
- *
- * + Clean up any existing Rust builds by running `cargo clean`.
- * + Run `cargo update` to make sure all dependencies are available.
- * + Compile rust code into index.node file.
- * + Run unit tests to ensure the library is in good shape for publishing.
- * + Move all expected content into a `dist` directory.
- * + Generate a binary distrubtion in `bin-package`.
- * + Do a dry run of npm publishing via irish-pub or perform an actual publish step if `--publish` option is provided.
- */
-
-const fs = require("fs");
-const path = require("path");
-const shell = require("shelljs");
-
-// Fail this script if any of these commands fail
-shell.set("-e");
-// Ensure that our directory is set to the root of the repo
-const rootDirectory = path.dirname(process.argv[1]);
-shell.cd(rootDirectory);
-
-const npmPublish = process.argv.slice(2).indexOf("--npm-publish") !== -1;
-const distPath = "./dist";
-
-// Cleanup the previous build, if it exists
-shell.rm("-rf", distPath);
-shell.rm("-rf", "./bin-package");
-shell.rm("-rf", "./build");
-
-// Cleanup any previous Rust builds, update deps, and compile
-shell.exec("npm ci --ignore-scripts");
-shell.exec("npm run clean");
-shell.pushd("./native");
-shell.exec("cargo update");
-shell.popd();
-shell.exec("npm run compile");
-
-// shell.exec("npm test");
-shell.mkdir(distPath);
-buildJsLib(distPath);
-shell.cp("-r", ["package.json", "../../LICENSE"], distPath);
-
-// Add a NPM install script to the package.json that we push to NPM so that when consumers pull it down it
-// runs the expected node-pre-gyp step.
-const npmPackageJson = require(`${distPath}/package.json`);
-npmPackageJson.scripts.install = "node-pre-gyp install";
-fs.writeFileSync(`${distPath}/package.json`, JSON.stringify(npmPackageJson, null, 2));
-
-shell.mkdir("./bin-package");
-shell.cp("./native/index.node", "./bin-package");
-shell.exec("npm run package");
-var tgz = shell.exec("find ./build -name *.tar.gz");
-shell.cp(tgz, "./bin-package/");
-shell.pushd(distPath);
-
-shell.exec(npmPublish ? "npm publish --access public" : "echo 'Skipping publishing to npm...'");
-shell.popd();
-
-shell.echo("publish.js COMPLETE");
-
-
-/**
- * Takes care of building the js lib
- * @param {string} path Location into which build the lib
- */
-function buildJsLib(path) {
-  shell.exec("npx tsc");
-  shell.cp("-r", ["lib/bindings"], path);
-  shell.mv([`${path}/bindings/native.prod.js`], [`${path}/bindings/native.js`]);
-}

bindings/node/tsconfig.json

@@ -5,7 +5,7 @@
   "compilerOptions": {
     /* Basic Options */
     // "incremental": true,                   /* Enable incremental compilation */
-    "target": "es6",                          /* Specify ECMAScript target version: 'ES3' (default), 'ES5', 'ES2015', 'ES2016', 'ES2017', 'ES2018', 'ES2019' or 'ESNEXT'. */
+    "target": "ES2017",                          /* Specify ECMAScript target version: 'ES3' (default), 'ES5', 'ES2015', 'ES2016', 'ES2017', 'ES2018', 'ES2019' or 'ESNEXT'. */
     "module": "commonjs",                     /* Specify module code generation: 'none', 'commonjs', 'amd', 'system', 'umd', 'es2015', or 'ESNext'. */
     // "lib": [],                             /* Specify library files to be included in the compilation. */
     // "allowJs": true,                       /* Allow javascript files to be compiled. */