mii/tokenizers
1
0
Fork 0
mirror of https://github.com/mii443/tokenizers.git synced 2025-08-22 16:25:30 +00:00
Code Issues Packages Projects Releases Wiki Activity

Fixing clippy warnings on 1.71. (#1296)

Browse Source 
* Fixing clippy warnings on 1.71.

* Fix.

* Fmt.

* Python clippy.

* Should really set my env back again.

* Fix.
This commit is contained in:
Nicolas Patry
2023-07-16 15:58:38 +02:00
committed by GitHub
parent 4811f769a1
commit 291b2e23ae
11 changed files with 29 additions and 30 deletions
Download Patch File Download Diff File Expand all files Collapse all files

13
bindings/node/native/src/pre_tokenizers.rs
View File

@ -155,8 +155,7 @@ fn byte_level_alphabet(mut cx: FunctionContext) -> JsResult<JsValue> {
fn whitespace(mut cx: FunctionContext) -> JsResult<JsPreTokenizer> {
let mut pretok = JsPreTokenizer::new::<_, JsPreTokenizer, _>(&mut cx, vec![])?;
let guard = cx.lock();
pretok.borrow_mut(&guard).pretok =
Some(tk::pre_tokenizers::whitespace::Whitespace::default().into());
pretok.borrow_mut(&guard).pretok = Some(tk::pre_tokenizers::whitespace::Whitespace {}.into());
Ok(pretok)
}
@ -303,10 +302,10 @@ mod test {
#[test]
fn serialize() {
let js_wrapped: JsPreTokenizerWrapper = Whitespace::default().into();
let js_wrapped: JsPreTokenizerWrapper = Whitespace {}.into();
let js_ser = serde_json::to_string(&js_wrapped).unwrap();
let rs_wrapped = PreTokenizerWrapper::Whitespace(Whitespace::default());
let rs_wrapped = PreTokenizerWrapper::Whitespace(Whitespace {});
let rs_ser = serde_json::to_string(&rs_wrapped).unwrap();
assert_eq!(js_ser, rs_ser);
@ -320,11 +319,11 @@ mod test {
}
let js_seq: JsPreTokenizerWrapper =
Sequence::new(vec![WhitespaceSplit.into(), Whitespace::default().into()]).into();
Sequence::new(vec![WhitespaceSplit.into(), Whitespace {}.into()]).into();
let js_wrapper_ser = serde_json::to_string(&js_seq).unwrap();
let rs_wrapped = PreTokenizerWrapper::Sequence(Sequence::new(vec![
WhitespaceSplit.into(),
Whitespace::default().into(),
Whitespace {}.into(),
]));
let rs_ser = serde_json::to_string(&rs_wrapped).unwrap();
assert_eq!(js_wrapper_ser, rs_ser);
@ -335,7 +334,7 @@ mod test {
let js_ser = serde_json::to_string(&js_seq).unwrap();
assert_eq!(js_wrapper_ser, js_ser);
let rs_seq = Sequence::new(vec![WhitespaceSplit.into(), Whitespace::default().into()]);
let rs_seq = Sequence::new(vec![WhitespaceSplit.into(), Whitespace {}.into()]);
let rs_ser = serde_json::to_string(&rs_seq).unwrap();
assert_eq!(js_wrapper_ser, rs_ser);
}