mii/tokenizers
1
0
Fork 0
mirror of https://github.com/mii443/tokenizers.git synced 2025-08-23 00:35:35 +00:00
Code Issues Packages Projects Releases Wiki Activity

Node - Hotfix EncodeTask

Browse Source
This commit is contained in:
Anthony MOI
2020-01-09 16:05:48 -05:00
parent cb52b71f63
commit 6816628d1a
1 changed files with 2 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
bindings/node/native/src/tasks/tokenizer.rs
View File

@ -39,7 +39,7 @@ impl Task for EncodeTask {
fn perform(&self) -> Result<Self::Output, Self::Error> { fn perform(&self) -> Result<Self::Output, Self::Error> {
match self { match self {
EncodeTask::Single(worker, input) => { EncodeTask::Single(worker, input) => {
let mut input = unsafe { std::ptr::replace(&input as *const _ as *mut _, None) }; let mut input = unsafe { std::ptr::replace(input as *const _ as *mut _, None) };
let tokenizer: &Tokenizer = unsafe { &*worker.ptr }; let tokenizer: &Tokenizer = unsafe { &*worker.ptr };
tokenizer tokenizer
.encode(input.take().ok_or("No provided input")?) .encode(input.take().ok_or("No provided input")?)
@ -47,7 +47,7 @@ impl Task for EncodeTask {
.map(|encoding| EncodeOutput::Single(encoding)) .map(|encoding| EncodeOutput::Single(encoding))
} }
EncodeTask::Batch(worker, input) => { EncodeTask::Batch(worker, input) => {
let mut input = unsafe { std::ptr::replace(&input as *const _ as *mut _, None) }; let mut input = unsafe { std::ptr::replace(input as *const _ as *mut _, None) };
let tokenizer: &Tokenizer = unsafe { &*worker.ptr }; let tokenizer: &Tokenizer = unsafe { &*worker.ptr };
tokenizer tokenizer
.encode_batch(input.take().ok_or("No provided input")?) .encode_batch(input.take().ok_or("No provided input")?)