mirror of
https://github.com/mii443/tokenizers.git
synced 2025-08-27 02:29:38 +00:00
rust: derive Copy
for PaddingDirection
This commit is contained in:
@ -148,9 +148,10 @@ module.exports = {
|
|||||||
// ],
|
// ],
|
||||||
|
|
||||||
// An array of regexp pattern strings that are matched against all test paths, matched tests are skipped
|
// An array of regexp pattern strings that are matched against all test paths, matched tests are skipped
|
||||||
// testPathIgnorePatterns: [
|
testPathIgnorePatterns: [
|
||||||
// "/node_modules/"
|
"/node_modules/",
|
||||||
// ],
|
"/dist/"
|
||||||
|
],
|
||||||
|
|
||||||
// The regexp pattern or array of patterns that Jest uses to detect test files
|
// The regexp pattern or array of patterns that Jest uses to detect test files
|
||||||
// testRegex: [],
|
// testRegex: [],
|
||||||
|
@ -411,20 +411,6 @@ declare_types! {
|
|||||||
PaddingStrategy::BatchLongest
|
PaddingStrategy::BatchLongest
|
||||||
};
|
};
|
||||||
|
|
||||||
let params_object = JsObject::new(&mut cx);
|
|
||||||
if let Some(max_length) = max_length {
|
|
||||||
let obj_length = cx.number(max_length as f64);
|
|
||||||
params_object.set(&mut cx, "maxLength", obj_length).unwrap();
|
|
||||||
}
|
|
||||||
let obj_pad_id = cx.number(pad_id);
|
|
||||||
let obj_pad_type_id = cx.number(pad_type_id);
|
|
||||||
let obj_pad_token = cx.string(&pad_token);
|
|
||||||
let obj_direction = cx.string(&direction);
|
|
||||||
params_object.set(&mut cx, "padId", obj_pad_id).unwrap();
|
|
||||||
params_object.set(&mut cx, "padTypeId", obj_pad_type_id).unwrap();
|
|
||||||
params_object.set(&mut cx, "padToken", obj_pad_token).unwrap();
|
|
||||||
params_object.set(&mut cx, "direction", obj_direction).unwrap();
|
|
||||||
|
|
||||||
let mut this = cx.this();
|
let mut this = cx.this();
|
||||||
{
|
{
|
||||||
let guard = cx.lock();
|
let guard = cx.lock();
|
||||||
@ -438,6 +424,20 @@ declare_types! {
|
|||||||
}));
|
}));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
let params_object = JsObject::new(&mut cx);
|
||||||
|
if let Some(max_length) = max_length {
|
||||||
|
let obj_length = cx.number(max_length as f64);
|
||||||
|
params_object.set(&mut cx, "maxLength", obj_length).unwrap();
|
||||||
|
}
|
||||||
|
let obj_pad_id = cx.number(pad_id);
|
||||||
|
let obj_pad_type_id = cx.number(pad_type_id);
|
||||||
|
let obj_pad_token = cx.string(pad_token);
|
||||||
|
let obj_direction = cx.string(direction);
|
||||||
|
params_object.set(&mut cx, "padId", obj_pad_id).unwrap();
|
||||||
|
params_object.set(&mut cx, "padTypeId", obj_pad_type_id).unwrap();
|
||||||
|
params_object.set(&mut cx, "padToken", obj_pad_token).unwrap();
|
||||||
|
params_object.set(&mut cx, "direction", obj_direction).unwrap();
|
||||||
|
|
||||||
Ok(params_object.upcast())
|
Ok(params_object.upcast())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -1,7 +1,7 @@
|
|||||||
use crate::tokenizer::NormalizedString;
|
use crate::tokenizer::NormalizedString;
|
||||||
|
|
||||||
/// The various possible padding directions.
|
/// The various possible padding directions.
|
||||||
#[derive(Debug, Clone)]
|
#[derive(Debug, Clone, Copy)]
|
||||||
pub enum PaddingDirection {
|
pub enum PaddingDirection {
|
||||||
Left,
|
Left,
|
||||||
Right,
|
Right,
|
||||||
@ -219,7 +219,7 @@ impl Encoding {
|
|||||||
pad_id: u32,
|
pad_id: u32,
|
||||||
pad_type_id: u32,
|
pad_type_id: u32,
|
||||||
pad_token: &str,
|
pad_token: &str,
|
||||||
direction: &PaddingDirection,
|
direction: PaddingDirection,
|
||||||
) {
|
) {
|
||||||
if self.ids.len() > target_length {
|
if self.ids.len() > target_length {
|
||||||
// We just do nothing if the wanted padding length is smaller than us
|
// We just do nothing if the wanted padding length is smaller than us
|
||||||
|
@ -553,7 +553,7 @@ impl Tokenizer {
|
|||||||
params.pad_id,
|
params.pad_id,
|
||||||
params.pad_type_id,
|
params.pad_type_id,
|
||||||
¶ms.pad_token,
|
¶ms.pad_token,
|
||||||
¶ms.direction,
|
params.direction,
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -139,7 +139,7 @@ pub fn pad_encodings(
|
|||||||
params.pad_id,
|
params.pad_id,
|
||||||
params.pad_type_id,
|
params.pad_type_id,
|
||||||
¶ms.pad_token,
|
¶ms.pad_token,
|
||||||
¶ms.direction,
|
params.direction,
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
Reference in New Issue
Block a user