mirror of
https://github.com/mii443/tokenizers.git
synced 2025-08-23 00:35:35 +00:00
Node bindings backbone
This commit is contained in:
Anthony MOI
6
bindings/node/.gitignore
vendored
Normal file
6
bindings/node/.gitignore
vendored
Normal file
@ -0,0 +1,6 @@
|
|||||||
|
native/target
|
||||||
|
native/index.node
|
||||||
|
native/artifacts.json
|
||||||
|
**/*~
|
||||||
|
**/node_modules
|
||||||
|
**/.DS_Store
|
||||||
8
bindings/node/lib/index.js
Normal file
8
bindings/node/lib/index.js
Normal file
@ -0,0 +1,8 @@
|
|||||||
|
var addon = require('../native');
|
||||||
|
|
||||||
|
let s = "Hey man!";
|
||||||
|
if (typeof process.argv[2] == 'string') {
|
||||||
|
s = process.argv[2];
|
||||||
|
}
|
||||||
|
|
||||||
|
console.log(addon.tokenize(s));
|
||||||
139
bindings/node/native/Cargo.lock
generated
Normal file
139
bindings/node/native/Cargo.lock
generated
Normal file
@ -0,0 +1,139 @@
|
|||||||
|
# This file is automatically @generated by Cargo.
|
||||||
|
# It is not intended for manual editing.
|
||||||
|
[[package]]
|
||||||
|
name = "aho-corasick"
|
||||||
|
version = "0.7.6"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
dependencies = [
|
||||||
|
"memchr 2.2.1 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "cc"
|
||||||
|
version = "1.0.46"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "cfg-if"
|
||||||
|
version = "0.1.10"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "cslice"
|
||||||
|
version = "0.2.0"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "lazy_static"
|
||||||
|
version = "1.4.0"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "memchr"
|
||||||
|
version = "2.2.1"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "neon"
|
||||||
|
version = "0.3.3"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
dependencies = [
|
||||||
|
"cslice 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||||
|
"neon-build 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||||
|
"neon-runtime 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||||
|
"semver 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "neon-build"
|
||||||
|
version = "0.3.3"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
dependencies = [
|
||||||
|
"cfg-if 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||||
|
"neon-sys 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "neon-runtime"
|
||||||
|
version = "0.3.3"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
dependencies = [
|
||||||
|
"cfg-if 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||||
|
"neon-sys 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "neon-sys"
|
||||||
|
version = "0.3.3"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
dependencies = [
|
||||||
|
"cc 1.0.46 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||||
|
"regex 1.3.1 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "node"
|
||||||
|
version = "0.1.0"
|
||||||
|
dependencies = [
|
||||||
|
"neon 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||||
|
"neon-build 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||||
|
"tokenizers-lib 0.1.0",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "regex"
|
||||||
|
version = "1.3.1"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
dependencies = [
|
||||||
|
"aho-corasick 0.7.6 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||||
|
"memchr 2.2.1 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||||
|
"regex-syntax 0.6.12 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||||
|
"thread_local 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "regex-syntax"
|
||||||
|
version = "0.6.12"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "semver"
|
||||||
|
version = "0.9.0"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
dependencies = [
|
||||||
|
"semver-parser 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "semver-parser"
|
||||||
|
version = "0.7.0"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "thread_local"
|
||||||
|
version = "0.3.6"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
dependencies = [
|
||||||
|
"lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "tokenizers-lib"
|
||||||
|
version = "0.1.0"
|
||||||
|
|
||||||
|
[metadata]
|
||||||
|
"checksum aho-corasick 0.7.6 (registry+https://github.com/rust-lang/crates.io-index)" = "58fb5e95d83b38284460a5fda7d6470aa0b8844d283a0b614b8535e880800d2d"
|
||||||
|
"checksum cc 1.0.46 (registry+https://github.com/rust-lang/crates.io-index)" = "0213d356d3c4ea2c18c40b037c3be23cd639825c18f25ee670ac7813beeef99c"
|
||||||
|
"checksum cfg-if 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)" = "4785bdd1c96b2a846b2bd7cc02e86b6b3dbf14e7e53446c4f54c92a361040822"
|
||||||
|
"checksum cslice 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "697c714f50560202b1f4e2e09cd50a421881c83e9025db75d15f276616f04f40"
|
||||||
|
"checksum lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646"
|
||||||
|
"checksum memchr 2.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "88579771288728879b57485cc7d6b07d648c9f0141eb955f8ab7f9d45394468e"
|
||||||
|
"checksum neon 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)" = "53b85accbbd250627f899a6fc1f220bbb4c8c2ff6dc71830dc6b752b39c2eb97"
|
||||||
|
"checksum neon-build 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)" = "ae406bf1065c4399e69d328a3bd8d4f088f2a205dc3881bf68c0ac775bfef337"
|
||||||
|
"checksum neon-runtime 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)" = "d8465ac4ed3f340dead85e053b75a5f639f48ac6343b3523eff90a751758eead"
|
||||||
|
"checksum neon-sys 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)" = "8ae4cf3871ca5a395077e68144c1754e94e9e1e3329e7f8399d999ca573ed89a"
|
||||||
|
"checksum regex 1.3.1 (registry+https://github.com/rust-lang/crates.io-index)" = "dc220bd33bdce8f093101afe22a037b8eb0e5af33592e6a9caafff0d4cb81cbd"
|
||||||
|
"checksum regex-syntax 0.6.12 (registry+https://github.com/rust-lang/crates.io-index)" = "11a7e20d1cce64ef2fed88b66d347f88bd9babb82845b2b858f3edbf59a4f716"
|
||||||
|
"checksum semver 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)" = "1d7eb9ef2c18661902cc47e535f9bc51b78acd254da71d375c2f6720d9a40403"
|
||||||
|
"checksum semver-parser 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)" = "388a1df253eca08550bef6c72392cfe7c30914bf41df5269b68cbd6ff8f570a3"
|
||||||
|
"checksum thread_local 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)" = "c6b53e329000edc2b34dbe8545fd20e55a333362d0a321909685a19bd28c3f1b"
|
||||||
18
bindings/node/native/Cargo.toml
Normal file
18
bindings/node/native/Cargo.toml
Normal file
@ -0,0 +1,18 @@
|
|||||||
|
[package]
|
||||||
|
name = "node"
|
||||||
|
version = "0.1.0"
|
||||||
|
authors = ["Anthony MOI <m.anthony.moi@gmail.com>"]
|
||||||
|
license = "Apache-2.0"
|
||||||
|
build = "build.rs"
|
||||||
|
exclude = ["artifacts.json", "index.node"]
|
||||||
|
|
||||||
|
[lib]
|
||||||
|
name = "node"
|
||||||
|
crate-type = ["cdylib"]
|
||||||
|
|
||||||
|
[build-dependencies]
|
||||||
|
neon-build = "0.3.3"
|
||||||
|
|
||||||
|
[dependencies]
|
||||||
|
neon = "0.3.3"
|
||||||
|
tokenizers-lib = { path = "../../../tokenizers" }
|
||||||
7
bindings/node/native/build.rs
Normal file
7
bindings/node/native/build.rs
Normal file
@ -0,0 +1,7 @@
|
|||||||
|
extern crate neon_build;
|
||||||
|
|
||||||
|
fn main() {
|
||||||
|
neon_build::setup(); // must be called in build.rs
|
||||||
|
|
||||||
|
// add project-specific build logic here...
|
||||||
|
}
|
||||||
19
bindings/node/native/src/lib.rs
Normal file
19
bindings/node/native/src/lib.rs
Normal file
@ -0,0 +1,19 @@
|
|||||||
|
#[macro_use]
|
||||||
|
extern crate neon;
|
||||||
|
extern crate tokenizers as tk;
|
||||||
|
|
||||||
|
use neon::prelude::*;
|
||||||
|
|
||||||
|
fn tokenize(mut cx: FunctionContext) -> JsResult<JsArray> {
|
||||||
|
let s = cx.argument::<JsString>(0)?.value();
|
||||||
|
println!("Tokenizing in rust: {:?}", s);
|
||||||
|
let result = tk::tokenize(&s);
|
||||||
|
let js_array = JsArray::new(&mut cx, result.len() as u32);
|
||||||
|
for (i, token) in result.iter().enumerate() {
|
||||||
|
let n = cx.number(*token as f64);
|
||||||
|
js_array.set(&mut cx, i as u32, n)?;
|
||||||
|
}
|
||||||
|
Ok(js_array)
|
||||||
|
}
|
||||||
|
|
||||||
|
register_module!(mut cx, { cx.export_function("tokenize", tokenize) });
|
||||||
18
bindings/node/package.json
Normal file
18
bindings/node/package.json
Normal file
@ -0,0 +1,18 @@
|
|||||||
|
{
|
||||||
|
"name": "node",
|
||||||
|
"version": "0.1.0",
|
||||||
|
"description": "",
|
||||||
|
"main": "lib/index.js",
|
||||||
|
"repository": {
|
||||||
|
"type": "git",
|
||||||
|
"url": ".."
|
||||||
|
},
|
||||||
|
"author": "Anthony MOI <m.anthony.moi@gmail.com>",
|
||||||
|
"license": "Apache-2.0",
|
||||||
|
"dependencies": {
|
||||||
|
"neon-cli": "^0.3.3"
|
||||||
|
},
|
||||||
|
"scripts": {
|
||||||
|
"install": "neon build"
|
||||||
|
}
|
||||||
|
}
|
||||||
Reference in New Issue
Block a user