mii/tokenizers
1
0
Fork 0
mirror of https://github.com/mii443/tokenizers.git synced 2025-08-30 20:19:21 +00:00
Code Issues Packages Projects Releases Wiki Activity

Node - Fix EncodeTask

Browse Source
This commit is contained in:
Anthony MOI
2020-01-09 14:04:20 -05:00
parent bda03ffe8c
commit de35bb4f45
2 changed files with 3 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
bindings/node/native/src/tasks/tokenizer.rs
View File

@ -39,7 +39,7 @@ impl Task for EncodeTask {
fn perform(&self) -> Result<Self::Output, Self::Error> { fn perform(&self) -> Result<Self::Output, Self::Error> {
match self { match self {
EncodeTask::Single(worker, input) => { EncodeTask::Single(worker, input) => {
let mut input = unsafe { std::ptr::read(input) }; let mut input = unsafe { std::ptr::replace(&input as *const _ as *mut _, None) };
let tokenizer: &Tokenizer = unsafe { &*worker.ptr }; let tokenizer: &Tokenizer = unsafe { &*worker.ptr };
tokenizer tokenizer
.encode(input.take().ok_or("No provided input")?) .encode(input.take().ok_or("No provided input")?)
@ -47,7 +47,7 @@ impl Task for EncodeTask {
.map(|encoding| EncodeOutput::Single(encoding)) .map(|encoding| EncodeOutput::Single(encoding))
} }
EncodeTask::Batch(worker, input) => { EncodeTask::Batch(worker, input) => {
let mut input = unsafe { std::ptr::read(input) }; let mut input = unsafe { std::ptr::replace(&input as *const _ as *mut _, None) };
let tokenizer: &Tokenizer = unsafe { &*worker.ptr }; let tokenizer: &Tokenizer = unsafe { &*worker.ptr };
tokenizer tokenizer
.encode_batch(input.take().ok_or("No provided input")?) .encode_batch(input.take().ok_or("No provided input")?)

3
bindings/node/native/src/tokenizer.rs
View File

@ -1,6 +1,5 @@
extern crate tokenizers as tk; extern crate tokenizers as tk;
use crate::encoding::*;
use crate::models::*; use crate::models::*;
use crate::tasks::tokenizer::{EncodeTask, WorkingTokenizer}; use crate::tasks::tokenizer::{EncodeTask, WorkingTokenizer};
use neon::prelude::*; use neon::prelude::*;
@ -16,7 +15,7 @@ pub struct Tokenizer {
impl Tokenizer { impl Tokenizer {
pub fn prepare_for_task(&self) -> Option<WorkingTokenizer> { pub fn prepare_for_task(&self) -> Option<WorkingTokenizer> {
if std::sync::Arc::strong_count(&self.running_task) > 0 { if std::sync::Arc::strong_count(&self.running_task) > 1 {
None None
} else { } else {
unsafe { unsafe {