mirror of
https://github.com/mii443/tokenizers.git
synced 2025-12-12 05:18:39 +00:00
New clippy comments?
This commit is contained in:
@@ -142,9 +142,9 @@ impl PyMetaspaceDec {
|
||||
match key {
|
||||
"replacement" => {
|
||||
let s: &str = value.extract()?;
|
||||
replacement = s.chars().nth(0).ok_or(exceptions::Exception::py_err(
|
||||
"replacement must be a character",
|
||||
))?;
|
||||
replacement = s.chars().next().ok_or_else(|| {
|
||||
exceptions::Exception::py_err("replacement must be a character")
|
||||
})?;
|
||||
}
|
||||
"add_prefix_space" => add_prefix_space = value.extract()?,
|
||||
_ => println!("Ignored unknown kwarg option {}", key),
|
||||
|
||||
@@ -181,10 +181,9 @@ impl PyEncoding {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Ok(self
|
||||
.encoding
|
||||
.pad(length, pad_id, pad_type_id, pad_token, direction))
|
||||
self.encoding
|
||||
.pad(length, pad_id, pad_type_id, pad_token, direction);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[args(kwargs = "**")]
|
||||
@@ -200,7 +199,7 @@ impl PyEncoding {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Ok(self.encoding.truncate(max_length, stride))
|
||||
self.encoding.truncate(max_length, stride);
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
@@ -170,9 +170,7 @@ impl PySequence {
|
||||
for n in normalizers.iter() {
|
||||
let normalizer: PyRef<PyNormalizer> = n.extract()?;
|
||||
match &normalizer.normalizer {
|
||||
PyNormalizerWrapper::Sequence(inner) => {
|
||||
sequence.extend(inner.iter().map(|i| i.clone()))
|
||||
}
|
||||
PyNormalizerWrapper::Sequence(inner) => sequence.extend(inner.iter().cloned()),
|
||||
PyNormalizerWrapper::Wrapped(inner) => sequence.push(inner.clone()),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -187,10 +187,8 @@ impl PyCharDelimiterSplit {
|
||||
pub fn new(delimiter: &str) -> PyResult<(Self, PyPreTokenizer)> {
|
||||
let chr_delimiter = delimiter
|
||||
.chars()
|
||||
.nth(0)
|
||||
.ok_or(exceptions::Exception::py_err(
|
||||
"delimiter must be a single character",
|
||||
))?;
|
||||
.next()
|
||||
.ok_or_else(|| exceptions::Exception::py_err("delimiter must be a single character"))?;
|
||||
Ok((
|
||||
PyCharDelimiterSplit {},
|
||||
CharDelimiterSplit::new(chr_delimiter).into(),
|
||||
@@ -232,9 +230,7 @@ impl PySequence {
|
||||
for n in pre_tokenizers.iter() {
|
||||
let pretokenizer: PyRef<PyPreTokenizer> = n.extract()?;
|
||||
match &pretokenizer.pretok {
|
||||
PyPreTokenizerWrapper::Sequence(inner) => {
|
||||
sequence.extend(inner.iter().map(|i| i.clone()))
|
||||
}
|
||||
PyPreTokenizerWrapper::Sequence(inner) => sequence.extend(inner.iter().cloned()),
|
||||
PyPreTokenizerWrapper::Wrapped(inner) => sequence.push(inner.clone()),
|
||||
PyPreTokenizerWrapper::Custom(_) => unreachable!(
|
||||
"Custom pretokenizers are currently disabled, how did you get here?"
|
||||
@@ -268,9 +264,9 @@ impl PyMetaspace {
|
||||
match key {
|
||||
"replacement" => {
|
||||
let s: &str = value.extract()?;
|
||||
replacement = s.chars().nth(0).ok_or(exceptions::Exception::py_err(
|
||||
"replacement must be a character",
|
||||
))?;
|
||||
replacement = s.chars().next().ok_or_else(|| {
|
||||
exceptions::Exception::py_err("replacement must be a character")
|
||||
})?;
|
||||
}
|
||||
"add_prefix_space" => add_prefix_space = value.extract()?,
|
||||
_ => println!("Ignored unknown kwarg option {}", key),
|
||||
|
||||
@@ -254,7 +254,7 @@ impl FromPyObject<'_> for PyArrayStr {
|
||||
let objects = std::slice::from_raw_parts(data as *const PyObject, n_elem);
|
||||
|
||||
let seq = objects
|
||||
.into_iter()
|
||||
.iter()
|
||||
.map(|obj| {
|
||||
let gil = Python::acquire_gil();
|
||||
let py = gil.python();
|
||||
|
||||
@@ -87,7 +87,7 @@ impl PyBpeTrainer {
|
||||
builder = builder.initial_alphabet(
|
||||
alphabet
|
||||
.into_iter()
|
||||
.map(|s| s.chars().nth(0))
|
||||
.map(|s| s.chars().next())
|
||||
.filter(|c| c.is_some())
|
||||
.map(|c| c.unwrap())
|
||||
.collect(),
|
||||
@@ -151,7 +151,7 @@ impl PyWordPieceTrainer {
|
||||
builder = builder.initial_alphabet(
|
||||
alphabet
|
||||
.into_iter()
|
||||
.map(|s| s.chars().nth(0))
|
||||
.map(|s| s.chars().next())
|
||||
.filter(|c| c.is_some())
|
||||
.map(|c| c.unwrap())
|
||||
.collect(),
|
||||
|
||||
Reference in New Issue
Block a user