mirror of
https://github.com/mii443/tokenizers.git
synced 2025-08-23 00:35:35 +00:00
rust: derive Copy
for PaddingDirection
This commit is contained in:
@ -148,9 +148,10 @@ module.exports = {
|
||||
// ],
|
||||
|
||||
// An array of regexp pattern strings that are matched against all test paths, matched tests are skipped
|
||||
// testPathIgnorePatterns: [
|
||||
// "/node_modules/"
|
||||
// ],
|
||||
testPathIgnorePatterns: [
|
||||
"/node_modules/",
|
||||
"/dist/"
|
||||
],
|
||||
|
||||
// The regexp pattern or array of patterns that Jest uses to detect test files
|
||||
// testRegex: [],
|
||||
|
@ -411,20 +411,6 @@ declare_types! {
|
||||
PaddingStrategy::BatchLongest
|
||||
};
|
||||
|
||||
let params_object = JsObject::new(&mut cx);
|
||||
if let Some(max_length) = max_length {
|
||||
let obj_length = cx.number(max_length as f64);
|
||||
params_object.set(&mut cx, "maxLength", obj_length).unwrap();
|
||||
}
|
||||
let obj_pad_id = cx.number(pad_id);
|
||||
let obj_pad_type_id = cx.number(pad_type_id);
|
||||
let obj_pad_token = cx.string(&pad_token);
|
||||
let obj_direction = cx.string(&direction);
|
||||
params_object.set(&mut cx, "padId", obj_pad_id).unwrap();
|
||||
params_object.set(&mut cx, "padTypeId", obj_pad_type_id).unwrap();
|
||||
params_object.set(&mut cx, "padToken", obj_pad_token).unwrap();
|
||||
params_object.set(&mut cx, "direction", obj_direction).unwrap();
|
||||
|
||||
let mut this = cx.this();
|
||||
{
|
||||
let guard = cx.lock();
|
||||
@ -438,6 +424,20 @@ declare_types! {
|
||||
}));
|
||||
}
|
||||
|
||||
let params_object = JsObject::new(&mut cx);
|
||||
if let Some(max_length) = max_length {
|
||||
let obj_length = cx.number(max_length as f64);
|
||||
params_object.set(&mut cx, "maxLength", obj_length).unwrap();
|
||||
}
|
||||
let obj_pad_id = cx.number(pad_id);
|
||||
let obj_pad_type_id = cx.number(pad_type_id);
|
||||
let obj_pad_token = cx.string(pad_token);
|
||||
let obj_direction = cx.string(direction);
|
||||
params_object.set(&mut cx, "padId", obj_pad_id).unwrap();
|
||||
params_object.set(&mut cx, "padTypeId", obj_pad_type_id).unwrap();
|
||||
params_object.set(&mut cx, "padToken", obj_pad_token).unwrap();
|
||||
params_object.set(&mut cx, "direction", obj_direction).unwrap();
|
||||
|
||||
Ok(params_object.upcast())
|
||||
}
|
||||
|
||||
|
@ -1,7 +1,7 @@
|
||||
use crate::tokenizer::NormalizedString;
|
||||
|
||||
/// The various possible padding directions.
|
||||
#[derive(Debug, Clone)]
|
||||
#[derive(Debug, Clone, Copy)]
|
||||
pub enum PaddingDirection {
|
||||
Left,
|
||||
Right,
|
||||
@ -219,7 +219,7 @@ impl Encoding {
|
||||
pad_id: u32,
|
||||
pad_type_id: u32,
|
||||
pad_token: &str,
|
||||
direction: &PaddingDirection,
|
||||
direction: PaddingDirection,
|
||||
) {
|
||||
if self.ids.len() > target_length {
|
||||
// We just do nothing if the wanted padding length is smaller than us
|
||||
|
@ -553,7 +553,7 @@ impl Tokenizer {
|
||||
params.pad_id,
|
||||
params.pad_type_id,
|
||||
¶ms.pad_token,
|
||||
¶ms.direction,
|
||||
params.direction,
|
||||
);
|
||||
}
|
||||
|
||||
|
@ -139,7 +139,7 @@ pub fn pad_encodings(
|
||||
params.pad_id,
|
||||
params.pad_type_id,
|
||||
¶ms.pad_token,
|
||||
¶ms.direction,
|
||||
params.direction,
|
||||
);
|
||||
}
|
||||
|
||||
|
Reference in New Issue
Block a user