mii/tokenizers
1
0
Fork 0
mirror of https://github.com/mii443/tokenizers.git synced 2025-12-09 06:08:22 +00:00
Code Issues Packages Projects Releases Wiki Activity

Node - Update for new InputSequence

Browse Source
This commit is contained in:
Anthony MOI
2020-08-21 18:15:37 -04:00
committed by Anthony MOI
parent ac1dc4b842
commit 7a93e9a54d
2 changed files with 20 additions and 20 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
bindings/node/native/src/tasks/tokenizer.rs
View File

@@ -5,9 +5,9 @@ use crate::tokenizer::Tokenizer;
use neon::prelude::*;
use tk::tokenizer::{EncodeInput, Encoding};
pub enum EncodeTask {
Single(Tokenizer, Option<EncodeInput>, bool),
Batch(Tokenizer, Option<Vec<EncodeInput>>, bool),
pub enum EncodeTask<'s> {
Single(Tokenizer, Option<EncodeInput<'s>>, bool),
Batch(Tokenizer, Option<Vec<EncodeInput<'s>>>, bool),
}
pub enum EncodeOutput {
@@ -15,7 +15,7 @@ pub enum EncodeOutput {
Batch(Vec<Encoding>),
}
impl Task for EncodeTask {
impl Task for EncodeTask<'static> {
type Output = EncodeOutput;
type Error = String;
type JsEvent = JsValue;

32
bindings/node/native/src/tokenizer.rs
View File

@@ -124,9 +124,9 @@ impl FromJsValue for SpecialToken {
// encode & encodeBatch types
struct TextInputSequence(tk::InputSequence);
struct PreTokenizedInputSequence(tk::InputSequence);
impl FromJsValue for PreTokenizedInputSequence {
struct TextInputSequence<'s>(tk::InputSequence<'s>);
struct PreTokenizedInputSequence<'s>(tk::InputSequence<'s>);
impl FromJsValue for PreTokenizedInputSequence<'_> {
fn from_value<'c, C: Context<'c>>(from: Handle<'c, JsValue>, cx: &mut C) -> LibResult<Self> {
let sequence = from
.downcast::<JsArray>()?
@@ -137,25 +137,25 @@ impl FromJsValue for PreTokenizedInputSequence {
Ok(Self(sequence.into()))
}
}
impl From<PreTokenizedInputSequence> for tk::InputSequence {
fn from(v: PreTokenizedInputSequence) -> Self {
impl<'s> From<PreTokenizedInputSequence<'s>> for tk::InputSequence<'s> {
fn from(v: PreTokenizedInputSequence<'s>) -> Self {
v.0
}
}
impl FromJsValue for TextInputSequence {
impl FromJsValue for TextInputSequence<'_> {
fn from_value<'c, C: Context<'c>>(from: Handle<'c, JsValue>, _cx: &mut C) -> LibResult<Self> {
Ok(Self(from.downcast::<JsString>()?.value().into()))
}
}
impl From<TextInputSequence> for tk::InputSequence {
fn from(v: TextInputSequence) -> Self {
impl<'s> From<TextInputSequence<'s>> for tk::InputSequence<'s> {
fn from(v: TextInputSequence<'s>) -> Self {
v.0
}
}
struct TextEncodeInput(tk::EncodeInput);
struct PreTokenizedEncodeInput(tk::EncodeInput);
impl FromJsValue for PreTokenizedEncodeInput {
struct TextEncodeInput<'s>(tk::EncodeInput<'s>);
struct PreTokenizedEncodeInput<'s>(tk::EncodeInput<'s>);
impl FromJsValue for PreTokenizedEncodeInput<'_> {
fn from_value<'c, C: Context<'c>>(from: Handle<'c, JsValue>, cx: &mut C) -> LibResult<Self> {
// If array is of size 2, and the first element is also an array, we'll parse a pair
let array = from.downcast::<JsArray>()?;
@@ -177,12 +177,12 @@ impl FromJsValue for PreTokenizedEncodeInput {
}
}
}
impl From<PreTokenizedEncodeInput> for tk::EncodeInput {
fn from(v: PreTokenizedEncodeInput) -> Self {
impl<'s> From<PreTokenizedEncodeInput<'s>> for tk::EncodeInput<'s> {
fn from(v: PreTokenizedEncodeInput<'s>) -> Self {
v.0
}
}
impl FromJsValue for TextEncodeInput {
impl FromJsValue for TextEncodeInput<'_> {
fn from_value<'c, C: Context<'c>>(from: Handle<'c, JsValue>, cx: &mut C) -> LibResult<Self> {
// If we get an array, it's a pair of sequences
if let Ok(array) = from.downcast::<JsArray>() {
@@ -204,8 +204,8 @@ impl FromJsValue for TextEncodeInput {
}
}
}
impl From<TextEncodeInput> for tk::EncodeInput {
fn from(v: TextEncodeInput) -> Self {
impl<'s> From<TextEncodeInput<'s>> for tk::EncodeInput<'s> {
fn from(v: TextEncodeInput<'s>) -> Self {
v.0
}
}