diff --git a/bindings/node/native/src/tasks/tokenizer.rs b/bindings/node/native/src/tasks/tokenizer.rs index 1b6377c4..8ac8d751 100644 --- a/bindings/node/native/src/tasks/tokenizer.rs +++ b/bindings/node/native/src/tasks/tokenizer.rs @@ -5,9 +5,9 @@ use crate::tokenizer::Tokenizer; use neon::prelude::*; use tk::tokenizer::{EncodeInput, Encoding}; -pub enum EncodeTask { - Single(Tokenizer, Option, bool), - Batch(Tokenizer, Option>, bool), +pub enum EncodeTask<'s> { + Single(Tokenizer, Option>, bool), + Batch(Tokenizer, Option>>, bool), } pub enum EncodeOutput { @@ -15,7 +15,7 @@ pub enum EncodeOutput { Batch(Vec), } -impl Task for EncodeTask { +impl Task for EncodeTask<'static> { type Output = EncodeOutput; type Error = String; type JsEvent = JsValue; diff --git a/bindings/node/native/src/tokenizer.rs b/bindings/node/native/src/tokenizer.rs index 4a2f9a98..f3f25fda 100644 --- a/bindings/node/native/src/tokenizer.rs +++ b/bindings/node/native/src/tokenizer.rs @@ -124,9 +124,9 @@ impl FromJsValue for SpecialToken { // encode & encodeBatch types -struct TextInputSequence(tk::InputSequence); -struct PreTokenizedInputSequence(tk::InputSequence); -impl FromJsValue for PreTokenizedInputSequence { +struct TextInputSequence<'s>(tk::InputSequence<'s>); +struct PreTokenizedInputSequence<'s>(tk::InputSequence<'s>); +impl FromJsValue for PreTokenizedInputSequence<'_> { fn from_value<'c, C: Context<'c>>(from: Handle<'c, JsValue>, cx: &mut C) -> LibResult { let sequence = from .downcast::()? @@ -137,25 +137,25 @@ impl FromJsValue for PreTokenizedInputSequence { Ok(Self(sequence.into())) } } -impl From for tk::InputSequence { - fn from(v: PreTokenizedInputSequence) -> Self { +impl<'s> From> for tk::InputSequence<'s> { + fn from(v: PreTokenizedInputSequence<'s>) -> Self { v.0 } } -impl FromJsValue for TextInputSequence { +impl FromJsValue for TextInputSequence<'_> { fn from_value<'c, C: Context<'c>>(from: Handle<'c, JsValue>, _cx: &mut C) -> LibResult { Ok(Self(from.downcast::()?.value().into())) } } -impl From for tk::InputSequence { - fn from(v: TextInputSequence) -> Self { +impl<'s> From> for tk::InputSequence<'s> { + fn from(v: TextInputSequence<'s>) -> Self { v.0 } } -struct TextEncodeInput(tk::EncodeInput); -struct PreTokenizedEncodeInput(tk::EncodeInput); -impl FromJsValue for PreTokenizedEncodeInput { +struct TextEncodeInput<'s>(tk::EncodeInput<'s>); +struct PreTokenizedEncodeInput<'s>(tk::EncodeInput<'s>); +impl FromJsValue for PreTokenizedEncodeInput<'_> { fn from_value<'c, C: Context<'c>>(from: Handle<'c, JsValue>, cx: &mut C) -> LibResult { // If array is of size 2, and the first element is also an array, we'll parse a pair let array = from.downcast::()?; @@ -177,12 +177,12 @@ impl FromJsValue for PreTokenizedEncodeInput { } } } -impl From for tk::EncodeInput { - fn from(v: PreTokenizedEncodeInput) -> Self { +impl<'s> From> for tk::EncodeInput<'s> { + fn from(v: PreTokenizedEncodeInput<'s>) -> Self { v.0 } } -impl FromJsValue for TextEncodeInput { +impl FromJsValue for TextEncodeInput<'_> { fn from_value<'c, C: Context<'c>>(from: Handle<'c, JsValue>, cx: &mut C) -> LibResult { // If we get an array, it's a pair of sequences if let Ok(array) = from.downcast::() { @@ -204,8 +204,8 @@ impl FromJsValue for TextEncodeInput { } } } -impl From for tk::EncodeInput { - fn from(v: TextEncodeInput) -> Self { +impl<'s> From> for tk::EncodeInput<'s> { + fn from(v: TextEncodeInput<'s>) -> Self { v.0 } }