From 802e5b1ef9db6cf996f77f6a1fb85d2d7d6ad7bd Mon Sep 17 00:00:00 2001 From: Anthony MOI Date: Tue, 4 Aug 2020 11:04:57 -0400 Subject: [PATCH] Node - impl Serialize/Deserialize for new components --- bindings/node/native/Cargo.lock | 149 ++++++++++++--------- bindings/node/native/Cargo.toml | 3 +- bindings/node/native/src/decoders.rs | 3 +- bindings/node/native/src/lib.rs | 3 +- bindings/node/native/src/models.rs | 3 +- bindings/node/native/src/normalizers.rs | 23 +++- bindings/node/native/src/pre_tokenizers.rs | 3 +- bindings/node/native/src/processors.rs | 5 +- bindings/node/native/src/tokenizer.rs | 30 +++-- 9 files changed, 134 insertions(+), 88 deletions(-) diff --git a/bindings/node/native/Cargo.lock b/bindings/node/native/Cargo.lock index 3b3d7ae8..f85c2381 100644 --- a/bindings/node/native/Cargo.lock +++ b/bindings/node/native/Cargo.lock @@ -2,18 +2,24 @@ # It is not intended for manual editing. [[package]] name = "addr2line" -version = "0.12.1" +version = "0.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a49806b9dadc843c61e7c97e72490ad7f7220ae249012fbda9ad0609457c0543" +checksum = "1b6a2d3371669ab3ca9797670853d61402b03d0b4b9ebf33d677dfa720203072" dependencies = [ "gimli", ] [[package]] -name = "aho-corasick" -version = "0.7.10" +name = "adler" +version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8716408b8bc624ed7f65d223ddb9ac2d044c0547b6fa4b0d554f3a9540496ada" +checksum = "ee2a4ec343196209d6594e19543ae87a39f96d5534d7174822a3ad825dd6ed7e" + +[[package]] +name = "aho-corasick" +version = "0.7.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "043164d8ba5c4c3035fec9bbee8647c0261d788f3474306f93bb65901cae0e86" dependencies = [ "memchr", ] @@ -46,13 +52,14 @@ checksum = "f8aac770f1885fd7e387acedd76065302551364496e46b3dd00860b2f8359b9d" [[package]] name = "backtrace" -version = "0.3.48" +version = "0.3.50" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0df2f85c8a2abbe3b7d7e748052fdd9b76a0458fdeb16ad4223f5eca78c7c130" +checksum = "46254cf2fdcdf1badb5934448c1bcbe046a56537b3987d96c51a7afc5d03f293" dependencies = [ "addr2line", "cfg-if", "libc", + "miniz_oxide", "object", "rustc-demangle", ] @@ -65,9 +72,9 @@ checksum = "cf1de2fe8c75bc145a2f577add951f8134889b4795d47466a54a5c846d691693" [[package]] name = "cc" -version = "1.0.54" +version = "1.0.58" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7bbb73db36c1246e9034e307d0fba23f9a2e251faa47ade70c1bd252220c8311" +checksum = "f9a06fb2e53271d7c279ec1efea6ab691c35a2ae67ec0d91d7acec0caf13b518" [[package]] name = "cfg-if" @@ -135,12 +142,13 @@ dependencies = [ [[package]] name = "crossbeam-queue" -version = "0.2.1" +version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c695eeca1e7173472a32221542ae469b3e9aac3a4fc81f7696bcad82029493db" +checksum = "774ba60a54c213d409d5353bda12d49cd68d14e45036a285234c8d6f91f92570" dependencies = [ "cfg-if", "crossbeam-utils", + "maybe-uninit", ] [[package]] @@ -174,9 +182,9 @@ checksum = "a357d28ed41a50f9c765dbfe56cbc04a64e53e5fc58ba79fbc34c10ef3df831f" [[package]] name = "error-chain" -version = "0.12.2" +version = "0.12.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d371106cc88ffdfb1eabd7111e432da544f16f3e2d7bf1dfe8bf575f1df045cd" +checksum = "2d2f06b9cac1506ece98fe3231e3cc9c4410ec3d5b1f24ae1c8946f0742cdefc" dependencies = [ "backtrace", "version_check", @@ -195,15 +203,15 @@ dependencies = [ [[package]] name = "gimli" -version = "0.21.0" +version = "0.22.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bcc8e0c9bce37868955864dbecd2b1ab2bdf967e6f28066d65aaac620444b65c" +checksum = "aaf91faf136cb47367fa430cd46e37a788775e7fa104f8b4bcb3861dc389b724" [[package]] name = "hermit-abi" -version = "0.1.13" +version = "0.1.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91780f809e750b0a89f5544be56617ff6b1227ee485bcb06ebe10cdf89bd3b71" +checksum = "3deed196b6e7f9e44a2ae8d94225d80302d81208b1bb673fd21fe634645c85a9" dependencies = [ "libc", ] @@ -240,9 +248,9 @@ dependencies = [ [[package]] name = "itoa" -version = "0.4.5" +version = "0.4.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b8b7a7c0c47db5545ed3fef7468ee7bb5b74691498139e4b3f6a20685dc6dd8e" +checksum = "dc6f3ad7b9d11a0c00842ff8de1b60ee58661048eb8049ed33c73594f359d7e6" [[package]] name = "lazy_static" @@ -252,9 +260,9 @@ checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" [[package]] name = "libc" -version = "0.2.71" +version = "0.2.74" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9457b06509d27052635f90d6466700c65095fdf75409b3fbdd903e988b886f49" +checksum = "a2f02823cf78b754822df5f7f268fb59822e7296276d3e069d8e8cb26a14bd10" [[package]] name = "maybe-uninit" @@ -270,13 +278,22 @@ checksum = "3728d817d99e5ac407411fa471ff9800a778d88a24685968b36824eaf4bee400" [[package]] name = "memoffset" -version = "0.5.4" +version = "0.5.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b4fc2c02a7e374099d4ee95a193111f72d2110197fe200272371758f6c3643d8" +checksum = "c198b026e1bbf08a937e94c6c60f9ec4a2267f5b0d2eec9c1b21b061ce2be55f" dependencies = [ "autocfg", ] +[[package]] +name = "miniz_oxide" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be0f75932c1f6cfae3c04000e40114adf955636e19040f9c0a2c380702aa1c7f" +dependencies = [ + "adler", +] + [[package]] name = "neon" version = "0.3.3" @@ -341,7 +358,6 @@ dependencies = [ "neon-runtime", "neon-serde", "serde", - "serde_derive", "tokenizers", ] @@ -370,9 +386,9 @@ dependencies = [ [[package]] name = "num-integer" -version = "0.1.42" +version = "0.1.43" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f6ea62e9d81a77cd3ee9a2a5b9b609447857f3d358704331e4ef39eb247fcba" +checksum = "8d59457e662d541ba17869cf51cf177c0b5f0cbf476c66bdc90bf1edac4f875b" dependencies = [ "autocfg", "num-traits", @@ -380,9 +396,9 @@ dependencies = [ [[package]] name = "num-iter" -version = "0.1.40" +version = "0.1.41" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dfb0800a0291891dd9f4fe7bd9c19384f98f7fbe0cd0f39a2c6b88b9868bbc00" +checksum = "7a6e6b7c748f995c4c29c5f5ae0248536e04a5739927c74ec0fa564805094b9f" dependencies = [ "autocfg", "num-integer", @@ -402,9 +418,9 @@ dependencies = [ [[package]] name = "num-traits" -version = "0.2.11" +version = "0.2.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c62be47e61d1842b9170f0fdeec8eba98e60e90e5446449a0545e5152acd7096" +checksum = "ac267bcc07f48ee5f8935ab0d24f316fb722d7a1292e2913f0cc196b29ffd611" dependencies = [ "autocfg", ] @@ -427,9 +443,9 @@ checksum = "17b02fc0ff9a9e4b35b3342880f48e896ebf69f2967921fe8646bf5b7125956a" [[package]] name = "object" -version = "0.19.0" +version = "0.20.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9cbca9424c482ee628fa549d9c812e2cd22f1180b9222c9200fdfa6eb31aecb2" +checksum = "1ab52be62400ca80aa00285d25253d7f7c437b7375c4de678f5405d3afe82ca5" [[package]] name = "onig" @@ -455,9 +471,9 @@ dependencies = [ [[package]] name = "pkg-config" -version = "0.3.17" +version = "0.3.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "05da548ad6865900e60eaba7f589cc0783590a92e940c26953ff81ddbab2d677" +checksum = "d36492546b6af1463394d46f0c834346f31548646f6ba10849802c9c9a27ac33" [[package]] name = "ppv-lite86" @@ -467,18 +483,18 @@ checksum = "237a5ed80e274dbc66f86bd59c1e25edc039660be53194b5fe0a482e0f2612ea" [[package]] name = "proc-macro2" -version = "1.0.17" +version = "1.0.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1502d12e458c49a4c9cbff560d0fe0060c252bc29799ed94ca2ed4bb665a0101" +checksum = "04f5f085b5d71e2188cb8271e5da0161ad52c3f227a661a3c135fdf28e258b12" dependencies = [ "unicode-xid", ] [[package]] name = "quote" -version = "1.0.6" +version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "54a21852a652ad6f610c9510194f398ff6f8692e334fd1145fed931f7fbe44ea" +checksum = "aa563d17ecb180e500da1cfd2b028310ac758de548efdd203e18f283af693f37" dependencies = [ "proc-macro2", ] @@ -526,10 +542,11 @@ dependencies = [ [[package]] name = "rayon" -version = "1.3.0" +version = "1.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "db6ce3297f9c85e16621bb8cca38a06779ffc31bb8184e1be4bed2be4678a098" +checksum = "62f02856753d04e03e26929f820d0a0a337ebe71f849801eea335d464b349080" dependencies = [ + "autocfg", "crossbeam-deque", "either", "rayon-core", @@ -547,9 +564,9 @@ dependencies = [ [[package]] name = "rayon-core" -version = "1.7.0" +version = "1.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08a89b46efaf957e52b18062fb2f4660f8b8a4dde1807ca002690868ef2c85a9" +checksum = "e92e15d89083484e11353891f1af602cc661426deb9564c298b270c726973280" dependencies = [ "crossbeam-deque", "crossbeam-queue", @@ -560,9 +577,9 @@ dependencies = [ [[package]] name = "regex" -version = "1.3.7" +version = "1.3.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a6020f034922e3194c711b82a627453881bc4682166cabb07134a10c26ba7692" +checksum = "9c3780fcf44b193bc4d09f36d2a3c87b251da4a046c87795a0d35f4f927ad8e6" dependencies = [ "aho-corasick", "memchr", @@ -572,9 +589,9 @@ dependencies = [ [[package]] name = "regex-syntax" -version = "0.6.17" +version = "0.6.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7fe5bd57d1d7414c6b5ed48563a2c855d995ff777729dcd91c369ec7fea395ae" +checksum = "26412eb97c6b088a6997e05f69403a802a92d520de2f8e63c2b65f9e0f47c4e8" [[package]] name = "rustc-demangle" @@ -584,9 +601,9 @@ checksum = "4c691c0e608126e00913e33f0ccf3727d5fc84573623b8d65b2df340b5201783" [[package]] name = "ryu" -version = "1.0.4" +version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ed3d612bc64430efeb3f7ee6ef26d590dce0c43249217bddc62112540c7941e1" +checksum = "71d301d4193d031abdd79ff7e3dd721168a9572ef3fe51a1517aba235bd8f86e" [[package]] name = "scopeguard" @@ -611,18 +628,18 @@ checksum = "388a1df253eca08550bef6c72392cfe7c30914bf41df5269b68cbd6ff8f570a3" [[package]] name = "serde" -version = "1.0.110" +version = "1.0.114" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "99e7b308464d16b56eba9964e4972a3eee817760ab60d88c3f86e1fecb08204c" +checksum = "5317f7588f0a5078ee60ef675ef96735a1442132dc645eb1d12c018620ed8cd3" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.110" +version = "1.0.114" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "818fbf6bfa9a42d3bfcaca148547aa00c7b915bec71d1757aa2d44ca68771984" +checksum = "2a0be94b04690fbaed37cddffc5c134bf537c8e3329d53e982fe04c374978f8e" dependencies = [ "proc-macro2", "quote", @@ -631,9 +648,9 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.53" +version = "1.0.57" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "993948e75b189211a9b31a7528f950c6adc21f9720b6438ff80a7fa2f864cea2" +checksum = "164eacbdb13512ec2745fb09d51fd5b22b0d65ed294a1dcf7285a360c80a675c" dependencies = [ "itoa", "ryu", @@ -642,9 +659,9 @@ dependencies = [ [[package]] name = "smallvec" -version = "1.4.0" +version = "1.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c7cb5678e1615754284ec264d9bb5b4c27d2018577fd90ac0ceb578591ed5ee4" +checksum = "3757cb9d89161a2f24e1cf78efa0c1fcff485d18e3f55e0aa3480824ddaa0f3f" [[package]] name = "strsim" @@ -654,9 +671,9 @@ checksum = "8ea5119cdb4c55b55d432abb513a0429384878c15dde60cc77b1c99de1a95a6a" [[package]] name = "syn" -version = "1.0.27" +version = "1.0.37" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ef781e621ee763a2a40721a8861ec519cb76966aee03bb5d00adb6a31dc1c1de" +checksum = "239f255b9e3429350f188c27b807fc9920a15eb9145230ff1a7d054c08fec319" dependencies = [ "proc-macro2", "quote", @@ -665,9 +682,9 @@ dependencies = [ [[package]] name = "terminal_size" -version = "0.1.12" +version = "0.1.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8038f95fc7a6f351163f4b964af631bd26c9e828f7db085f2a84aca56f70d13b" +checksum = "9a14cd9f8c72704232f0bfc8455c0e861f0ad4eb60cc9ec8a170e231414c1e13" dependencies = [ "libc", "winapi", @@ -731,15 +748,15 @@ dependencies = [ [[package]] name = "unicode-width" -version = "0.1.7" +version = "0.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "caaa9d531767d1ff2150b9332433f32a24622147e5ebb1f26409d5da67afd479" +checksum = "9337591893a19b88d8d87f2cec1e73fad5cdfd10e5a6f349f498ad6ea2ffb1e3" [[package]] name = "unicode-xid" -version = "0.2.0" +version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "826e7639553986605ec5979c7dd957c7895e93eabed50ab2ffa7f6128a75097c" +checksum = "f7fe0bb3479651439c9112f72b6c505038574c9fbb575ed1bf3b797fa39dd564" [[package]] name = "unicode_categories" @@ -767,9 +784,9 @@ checksum = "cccddf32554fecc6acb585f82a32a72e28b48f8c4c1883ddfeeeaa96f7d8e519" [[package]] name = "winapi" -version = "0.3.8" +version = "0.3.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8093091eeb260906a183e6ae1abdba2ef5ef2257a21801128899c3fc699229c6" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" dependencies = [ "winapi-i686-pc-windows-gnu", "winapi-x86_64-pc-windows-gnu", diff --git a/bindings/node/native/Cargo.toml b/bindings/node/native/Cargo.toml index 58540874..1f488081 100644 --- a/bindings/node/native/Cargo.toml +++ b/bindings/node/native/Cargo.toml @@ -17,6 +17,5 @@ neon-build = "0.3.3" neon = "0.3" neon-runtime = "0.3" neon-serde = "0.3" -serde = "1.0" -serde_derive = "1.0" +serde = { version = "1.0", features = [ "rc", "derive" ] } tokenizers = { path = "../../../tokenizers" } diff --git a/bindings/node/native/src/decoders.rs b/bindings/node/native/src/decoders.rs index da234c1a..c9a347c5 100644 --- a/bindings/node/native/src/decoders.rs +++ b/bindings/node/native/src/decoders.rs @@ -7,8 +7,9 @@ use std::sync::Arc; use tk::decoders::DecoderWrapper; /// Decoder -#[derive(Clone)] +#[derive(Clone, Serialize, Deserialize)] pub struct Decoder { + #[serde(flatten)] pub decoder: Option>, } diff --git a/bindings/node/native/src/lib.rs b/bindings/node/native/src/lib.rs index 9eb58600..b1b44ca2 100644 --- a/bindings/node/native/src/lib.rs +++ b/bindings/node/native/src/lib.rs @@ -2,9 +2,8 @@ extern crate neon; extern crate neon_serde; -extern crate serde; #[macro_use] -extern crate serde_derive; +extern crate serde; extern crate tokenizers as tk; mod container; diff --git a/bindings/node/native/src/models.rs b/bindings/node/native/src/models.rs index adc95c04..2d367e2a 100644 --- a/bindings/node/native/src/models.rs +++ b/bindings/node/native/src/models.rs @@ -13,8 +13,9 @@ use tk::Model as ModelTrait; use tk::Token; /// Model -#[derive(Clone)] +#[derive(Clone, Serialize, Deserialize)] pub struct Model { + #[serde(flatten)] pub model: Option>, } diff --git a/bindings/node/native/src/normalizers.rs b/bindings/node/native/src/normalizers.rs index e15a161c..b0d1305b 100644 --- a/bindings/node/native/src/normalizers.rs +++ b/bindings/node/native/src/normalizers.rs @@ -2,17 +2,35 @@ extern crate tokenizers as tk; use crate::extraction::*; use neon::prelude::*; +use serde::{ser::SerializeStruct, Serialize, Serializer}; use std::sync::Arc; use tk::normalizers::NormalizerWrapper; use tk::NormalizedString; -#[derive(Clone)] +#[derive(Clone, Deserialize)] pub enum JsNormalizerWrapper { Sequence(Vec>), Wrapped(Arc), } +impl Serialize for JsNormalizerWrapper { + fn serialize(&self, serializer: S) -> Result<::Ok, ::Error> + where + S: Serializer, + { + match self { + JsNormalizerWrapper::Sequence(seq) => { + let mut ser = serializer.serialize_struct("Sequence", 2)?; + ser.serialize_field("type", "Sequence")?; + ser.serialize_field("normalizers", seq)?; + ser.end() + } + JsNormalizerWrapper::Wrapped(inner) => inner.serialize(serializer), + } + } +} + impl From for JsNormalizerWrapper where I: Into, @@ -23,8 +41,9 @@ where } /// Normalizer -#[derive(Clone)] +#[derive(Clone, Serialize, Deserialize)] pub struct Normalizer { + #[serde(flatten)] pub normalizer: Option, } diff --git a/bindings/node/native/src/pre_tokenizers.rs b/bindings/node/native/src/pre_tokenizers.rs index 0d611540..c602f17f 100644 --- a/bindings/node/native/src/pre_tokenizers.rs +++ b/bindings/node/native/src/pre_tokenizers.rs @@ -8,8 +8,9 @@ use tk::pre_tokenizers::PreTokenizerWrapper; use tk::PreTokenizedString; /// PreTokenizers -#[derive(Clone)] +#[derive(Clone, Serialize, Deserialize)] pub struct PreTokenizer { + #[serde(flatten)] pub pretok: Option>, } diff --git a/bindings/node/native/src/processors.rs b/bindings/node/native/src/processors.rs index 96179859..2bfed1da 100644 --- a/bindings/node/native/src/processors.rs +++ b/bindings/node/native/src/processors.rs @@ -8,8 +8,9 @@ use tk::processors::PostProcessorWrapper; use tk::Encoding; /// Processor -#[derive(Clone)] -pub struct PostProcessor { +#[derive(Clone, Serialize, Deserialize)] +pub struct Processor { + #[serde(flatten)] pub processor: Option>, } diff --git a/bindings/node/native/src/tokenizer.rs b/bindings/node/native/src/tokenizer.rs index 942de376..eadc81e2 100644 --- a/bindings/node/native/src/tokenizer.rs +++ b/bindings/node/native/src/tokenizer.rs @@ -371,8 +371,10 @@ declare_types! { let this = cx.this(); let guard = cx.lock(); - let s = this.borrow(&guard).tokenizer.to_string(pretty); - let s = s.map_err(|e| cx.throw_error::<_, ()>(format!("{}", e)).unwrap_err())?; + let s = this.borrow(&guard) + .tokenizer.read().unwrap() + .to_string(pretty) + .map_err(|e| Error(format!("{}", e)))?; Ok(cx.string(s).upcast()) } @@ -384,8 +386,10 @@ declare_types! { let this = cx.this(); let guard = cx.lock(); - let res = this.borrow(&guard).tokenizer.save(&path, pretty); - res.map_err(|e| cx.throw_error::<_, ()>(format!("{}", e)).unwrap_err())?; + this.borrow(&guard) + .tokenizer.read().unwrap() + .save(&path, pretty) + .map_err(|e| Error(format!("{}", e)))?; Ok(cx.undefined().upcast()) } @@ -958,13 +962,17 @@ declare_types! { pub fn tokenizer_from_string(mut cx: FunctionContext) -> JsResult { let s = cx.extract::(0)?; - let tokenizer: tk::tokenizer::TokenizerImpl = s - .parse() - .map_err(|e| cx.throw_error::<_, ()>(format!("{}", e)).unwrap_err())?; + let tokenizer: tk::tokenizer::TokenizerImpl< + Model, + Normalizer, + PreTokenizer, + Processor, + Decoder, + > = s.parse().map_err(|e| Error(format!("{}", e)))?; let mut js_tokenizer = JsTokenizer::new::<_, JsTokenizer, _>(&mut cx, vec![])?; let guard = cx.lock(); - js_tokenizer.borrow_mut(&guard).tokenizer = tokenizer; + js_tokenizer.borrow_mut(&guard).tokenizer = Arc::new(RwLock::new(tokenizer)); Ok(js_tokenizer) } @@ -972,12 +980,12 @@ pub fn tokenizer_from_string(mut cx: FunctionContext) -> JsResult { pub fn tokenizer_from_file(mut cx: FunctionContext) -> JsResult { let s = cx.extract::(0)?; - let tokenizer = tk::tokenizer::TokenizerImpl::from_file(s) - .map_err(|e| cx.throw_error::<_, ()>(format!("{}", e)).unwrap_err())?; + let tokenizer = + tk::tokenizer::TokenizerImpl::from_file(s).map_err(|e| Error(format!("{}", e)))?; let mut js_tokenizer = JsTokenizer::new::<_, JsTokenizer, _>(&mut cx, vec![])?; let guard = cx.lock(); - js_tokenizer.borrow_mut(&guard).tokenizer = tokenizer; + js_tokenizer.borrow_mut(&guard).tokenizer = Arc::new(RwLock::new(tokenizer)); Ok(js_tokenizer) }