mirror of
https://github.com/mii443/tokenizers.git
synced 2025-08-22 16:25:30 +00:00
Node bindings backbone
This commit is contained in:
Anthony MOI
6
bindings/node/.gitignore
vendored
Normal file
6
bindings/node/.gitignore
vendored
Normal file
@ -0,0 +1,6 @@
|
||||
native/target
|
||||
native/index.node
|
||||
native/artifacts.json
|
||||
**/*~
|
||||
**/node_modules
|
||||
**/.DS_Store
|
||||
8
bindings/node/lib/index.js
Normal file
8
bindings/node/lib/index.js
Normal file
@ -0,0 +1,8 @@
|
||||
var addon = require('../native');
|
||||
|
||||
let s = "Hey man!";
|
||||
if (typeof process.argv[2] == 'string') {
|
||||
s = process.argv[2];
|
||||
}
|
||||
|
||||
console.log(addon.tokenize(s));
|
||||
139
bindings/node/native/Cargo.lock
generated
Normal file
139
bindings/node/native/Cargo.lock
generated
Normal file
@ -0,0 +1,139 @@
|
||||
# This file is automatically @generated by Cargo.
|
||||
# It is not intended for manual editing.
|
||||
[[package]]
|
||||
name = "aho-corasick"
|
||||
version = "0.7.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
dependencies = [
|
||||
"memchr 2.2.1 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "cc"
|
||||
version = "1.0.46"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
|
||||
[[package]]
|
||||
name = "cfg-if"
|
||||
version = "0.1.10"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
|
||||
[[package]]
|
||||
name = "cslice"
|
||||
version = "0.2.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
|
||||
[[package]]
|
||||
name = "lazy_static"
|
||||
version = "1.4.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
|
||||
[[package]]
|
||||
name = "memchr"
|
||||
version = "2.2.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
|
||||
[[package]]
|
||||
name = "neon"
|
||||
version = "0.3.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
dependencies = [
|
||||
"cslice 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"neon-build 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"neon-runtime 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"semver 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "neon-build"
|
||||
version = "0.3.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
dependencies = [
|
||||
"cfg-if 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"neon-sys 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "neon-runtime"
|
||||
version = "0.3.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
dependencies = [
|
||||
"cfg-if 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"neon-sys 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "neon-sys"
|
||||
version = "0.3.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
dependencies = [
|
||||
"cc 1.0.46 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"regex 1.3.1 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "node"
|
||||
version = "0.1.0"
|
||||
dependencies = [
|
||||
"neon 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"neon-build 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"tokenizers-lib 0.1.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "regex"
|
||||
version = "1.3.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
dependencies = [
|
||||
"aho-corasick 0.7.6 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"memchr 2.2.1 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"regex-syntax 0.6.12 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"thread_local 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "regex-syntax"
|
||||
version = "0.6.12"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
|
||||
[[package]]
|
||||
name = "semver"
|
||||
version = "0.9.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
dependencies = [
|
||||
"semver-parser 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "semver-parser"
|
||||
version = "0.7.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
|
||||
[[package]]
|
||||
name = "thread_local"
|
||||
version = "0.3.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
dependencies = [
|
||||
"lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "tokenizers-lib"
|
||||
version = "0.1.0"
|
||||
|
||||
[metadata]
|
||||
"checksum aho-corasick 0.7.6 (registry+https://github.com/rust-lang/crates.io-index)" = "58fb5e95d83b38284460a5fda7d6470aa0b8844d283a0b614b8535e880800d2d"
|
||||
"checksum cc 1.0.46 (registry+https://github.com/rust-lang/crates.io-index)" = "0213d356d3c4ea2c18c40b037c3be23cd639825c18f25ee670ac7813beeef99c"
|
||||
"checksum cfg-if 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)" = "4785bdd1c96b2a846b2bd7cc02e86b6b3dbf14e7e53446c4f54c92a361040822"
|
||||
"checksum cslice 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "697c714f50560202b1f4e2e09cd50a421881c83e9025db75d15f276616f04f40"
|
||||
"checksum lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646"
|
||||
"checksum memchr 2.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "88579771288728879b57485cc7d6b07d648c9f0141eb955f8ab7f9d45394468e"
|
||||
"checksum neon 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)" = "53b85accbbd250627f899a6fc1f220bbb4c8c2ff6dc71830dc6b752b39c2eb97"
|
||||
"checksum neon-build 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)" = "ae406bf1065c4399e69d328a3bd8d4f088f2a205dc3881bf68c0ac775bfef337"
|
||||
"checksum neon-runtime 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)" = "d8465ac4ed3f340dead85e053b75a5f639f48ac6343b3523eff90a751758eead"
|
||||
"checksum neon-sys 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)" = "8ae4cf3871ca5a395077e68144c1754e94e9e1e3329e7f8399d999ca573ed89a"
|
||||
"checksum regex 1.3.1 (registry+https://github.com/rust-lang/crates.io-index)" = "dc220bd33bdce8f093101afe22a037b8eb0e5af33592e6a9caafff0d4cb81cbd"
|
||||
"checksum regex-syntax 0.6.12 (registry+https://github.com/rust-lang/crates.io-index)" = "11a7e20d1cce64ef2fed88b66d347f88bd9babb82845b2b858f3edbf59a4f716"
|
||||
"checksum semver 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)" = "1d7eb9ef2c18661902cc47e535f9bc51b78acd254da71d375c2f6720d9a40403"
|
||||
"checksum semver-parser 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)" = "388a1df253eca08550bef6c72392cfe7c30914bf41df5269b68cbd6ff8f570a3"
|
||||
"checksum thread_local 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)" = "c6b53e329000edc2b34dbe8545fd20e55a333362d0a321909685a19bd28c3f1b"
|
||||
18
bindings/node/native/Cargo.toml
Normal file
18
bindings/node/native/Cargo.toml
Normal file
@ -0,0 +1,18 @@
|
||||
[package]
|
||||
name = "node"
|
||||
version = "0.1.0"
|
||||
authors = ["Anthony MOI <m.anthony.moi@gmail.com>"]
|
||||
license = "Apache-2.0"
|
||||
build = "build.rs"
|
||||
exclude = ["artifacts.json", "index.node"]
|
||||
|
||||
[lib]
|
||||
name = "node"
|
||||
crate-type = ["cdylib"]
|
||||
|
||||
[build-dependencies]
|
||||
neon-build = "0.3.3"
|
||||
|
||||
[dependencies]
|
||||
neon = "0.3.3"
|
||||
tokenizers-lib = { path = "../../../tokenizers" }
|
||||
7
bindings/node/native/build.rs
Normal file
7
bindings/node/native/build.rs
Normal file
@ -0,0 +1,7 @@
|
||||
extern crate neon_build;
|
||||
|
||||
fn main() {
|
||||
neon_build::setup(); // must be called in build.rs
|
||||
|
||||
// add project-specific build logic here...
|
||||
}
|
||||
19
bindings/node/native/src/lib.rs
Normal file
19
bindings/node/native/src/lib.rs
Normal file
@ -0,0 +1,19 @@
|
||||
#[macro_use]
|
||||
extern crate neon;
|
||||
extern crate tokenizers as tk;
|
||||
|
||||
use neon::prelude::*;
|
||||
|
||||
fn tokenize(mut cx: FunctionContext) -> JsResult<JsArray> {
|
||||
let s = cx.argument::<JsString>(0)?.value();
|
||||
println!("Tokenizing in rust: {:?}", s);
|
||||
let result = tk::tokenize(&s);
|
||||
let js_array = JsArray::new(&mut cx, result.len() as u32);
|
||||
for (i, token) in result.iter().enumerate() {
|
||||
let n = cx.number(*token as f64);
|
||||
js_array.set(&mut cx, i as u32, n)?;
|
||||
}
|
||||
Ok(js_array)
|
||||
}
|
||||
|
||||
register_module!(mut cx, { cx.export_function("tokenize", tokenize) });
|
||||
18
bindings/node/package.json
Normal file
18
bindings/node/package.json
Normal file
@ -0,0 +1,18 @@
|
||||
{
|
||||
"name": "node",
|
||||
"version": "0.1.0",
|
||||
"description": "",
|
||||
"main": "lib/index.js",
|
||||
"repository": {
|
||||
"type": "git",
|
||||
"url": ".."
|
||||
},
|
||||
"author": "Anthony MOI <m.anthony.moi@gmail.com>",
|
||||
"license": "Apache-2.0",
|
||||
"dependencies": {
|
||||
"neon-cli": "^0.3.3"
|
||||
},
|
||||
"scripts": {
|
||||
"install": "neon build"
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user