mii/tokenizers
1
0
Fork 0
mirror of https://github.com/mii443/tokenizers.git synced 2025-12-12 21:38:35 +00:00
Code Issues Packages Projects Releases Wiki Activity

Node - Update for new InputSequence

Browse Source
This commit is contained in:
Anthony MOI
2020-08-21 18:15:37 -04:00
committed by Anthony MOI
parent ac1dc4b842
commit 7a93e9a54d
2 changed files with 20 additions and 20 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
bindings/node/native/src/tasks/tokenizer.rs
View File

@@ -5,9 +5,9 @@ use crate::tokenizer::Tokenizer;
use neon::prelude::*; use neon::prelude::*;
use tk::tokenizer::{EncodeInput, Encoding}; use tk::tokenizer::{EncodeInput, Encoding};
pub enum EncodeTask { pub enum EncodeTask<'s> {
Single(Tokenizer, Option<EncodeInput>, bool), Single(Tokenizer, Option<EncodeInput<'s>>, bool),
Batch(Tokenizer, Option<Vec<EncodeInput>>, bool), Batch(Tokenizer, Option<Vec<EncodeInput<'s>>>, bool),
} }
pub enum EncodeOutput { pub enum EncodeOutput {
@@ -15,7 +15,7 @@ pub enum EncodeOutput {
Batch(Vec<Encoding>), Batch(Vec<Encoding>),
} }
impl Task for EncodeTask { impl Task for EncodeTask<'static> {
type Output = EncodeOutput; type Output = EncodeOutput;
type Error = String; type Error = String;
type JsEvent = JsValue; type JsEvent = JsValue;

32
bindings/node/native/src/tokenizer.rs
View File

@@ -124,9 +124,9 @@ impl FromJsValue for SpecialToken {
// encode & encodeBatch types // encode & encodeBatch types
struct TextInputSequence(tk::InputSequence); struct TextInputSequence<'s>(tk::InputSequence<'s>);
struct PreTokenizedInputSequence(tk::InputSequence); struct PreTokenizedInputSequence<'s>(tk::InputSequence<'s>);
impl FromJsValue for PreTokenizedInputSequence { impl FromJsValue for PreTokenizedInputSequence<'_> {
fn from_value<'c, C: Context<'c>>(from: Handle<'c, JsValue>, cx: &mut C) -> LibResult<Self> { fn from_value<'c, C: Context<'c>>(from: Handle<'c, JsValue>, cx: &mut C) -> LibResult<Self> {
let sequence = from let sequence = from
.downcast::<JsArray>()? .downcast::<JsArray>()?
@@ -137,25 +137,25 @@ impl FromJsValue for PreTokenizedInputSequence {
Ok(Self(sequence.into())) Ok(Self(sequence.into()))
} }
} }
impl From<PreTokenizedInputSequence> for tk::InputSequence { impl<'s> From<PreTokenizedInputSequence<'s>> for tk::InputSequence<'s> {
fn from(v: PreTokenizedInputSequence) -> Self { fn from(v: PreTokenizedInputSequence<'s>) -> Self {
v.0 v.0
} }
} }
impl FromJsValue for TextInputSequence { impl FromJsValue for TextInputSequence<'_> {
fn from_value<'c, C: Context<'c>>(from: Handle<'c, JsValue>, _cx: &mut C) -> LibResult<Self> { fn from_value<'c, C: Context<'c>>(from: Handle<'c, JsValue>, _cx: &mut C) -> LibResult<Self> {
Ok(Self(from.downcast::<JsString>()?.value().into())) Ok(Self(from.downcast::<JsString>()?.value().into()))
} }
} }
impl From<TextInputSequence> for tk::InputSequence { impl<'s> From<TextInputSequence<'s>> for tk::InputSequence<'s> {
fn from(v: TextInputSequence) -> Self { fn from(v: TextInputSequence<'s>) -> Self {
v.0 v.0
} }
} }
struct TextEncodeInput(tk::EncodeInput); struct TextEncodeInput<'s>(tk::EncodeInput<'s>);
struct PreTokenizedEncodeInput(tk::EncodeInput); struct PreTokenizedEncodeInput<'s>(tk::EncodeInput<'s>);
impl FromJsValue for PreTokenizedEncodeInput { impl FromJsValue for PreTokenizedEncodeInput<'_> {
fn from_value<'c, C: Context<'c>>(from: Handle<'c, JsValue>, cx: &mut C) -> LibResult<Self> { fn from_value<'c, C: Context<'c>>(from: Handle<'c, JsValue>, cx: &mut C) -> LibResult<Self> {
// If array is of size 2, and the first element is also an array, we'll parse a pair // If array is of size 2, and the first element is also an array, we'll parse a pair
let array = from.downcast::<JsArray>()?; let array = from.downcast::<JsArray>()?;
@@ -177,12 +177,12 @@ impl FromJsValue for PreTokenizedEncodeInput {
} }
} }
} }
impl From<PreTokenizedEncodeInput> for tk::EncodeInput { impl<'s> From<PreTokenizedEncodeInput<'s>> for tk::EncodeInput<'s> {
fn from(v: PreTokenizedEncodeInput) -> Self { fn from(v: PreTokenizedEncodeInput<'s>) -> Self {
v.0 v.0
} }
} }
impl FromJsValue for TextEncodeInput { impl FromJsValue for TextEncodeInput<'_> {
fn from_value<'c, C: Context<'c>>(from: Handle<'c, JsValue>, cx: &mut C) -> LibResult<Self> { fn from_value<'c, C: Context<'c>>(from: Handle<'c, JsValue>, cx: &mut C) -> LibResult<Self> {
// If we get an array, it's a pair of sequences // If we get an array, it's a pair of sequences
if let Ok(array) = from.downcast::<JsArray>() { if let Ok(array) = from.downcast::<JsArray>() {
@@ -204,8 +204,8 @@ impl FromJsValue for TextEncodeInput {
} }
} }
} }
impl From<TextEncodeInput> for tk::EncodeInput { impl<'s> From<TextEncodeInput<'s>> for tk::EncodeInput<'s> {
fn from(v: TextEncodeInput) -> Self { fn from(v: TextEncodeInput<'s>) -> Self {
v.0 v.0
} }
} }