mii/tokenizers
1
0
Fork 0
mirror of https://github.com/mii443/tokenizers.git synced 2025-08-22 16:25:30 +00:00
Code Issues Packages Projects Releases Wiki Activity

Improved version. (#1154)

Browse Source 
* Improved version.

* Clippy.
This commit is contained in:
Nicolas Patry
2023-01-23 16:35:19 +01:00
committed by GitHub
parent d09241fba1
commit fa66caf0ab
1 changed files with 11 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files

15
tokenizers/src/utils/from_pretrained.rs
View File

@ -104,13 +104,20 @@ pub fn from_pretrained<S: AsRef<str>>(
) -> Result<PathBuf> {
let identifier: &str = identifier.as_ref();
let is_valid_char =
|x: char| x.is_alphanumeric() || x == '-' || x == '_' || x == '.' || x == '/';
let valid_chars = ['-', '_', '.', '/'];
let is_valid_char = |x: char| x.is_alphanumeric() || valid_chars.contains(&x);
let valid = identifier.chars().all(is_valid_char);
let valid_chars_stringified = valid_chars
.iter()
.fold(vec![], |mut buf, x| {
buf.push(format!("'{}'", x));
buf
})
.join(", "); // "'/', '-', '_', '.'"
if !valid {
return Err(format!(
"Model \"{}\" contains invalid characters, expected only alphanumeric or '/', '-', '_', '.'",
"Model \"{}\" contains invalid characters, expected only alphanumeric or {valid_chars_stringified}",
identifier
)
.into());
@ -122,7 +129,7 @@ pub fn from_pretrained<S: AsRef<str>>(
let valid_revision = revision.chars().all(is_valid_char);
if !valid_revision {
return Err(format!(
"Revision \"{}\" contains invalid characters, expected only alphanumeric or '/', '-', '_', '.'",
"Revision \"{}\" contains invalid characters, expected only alphanumeric or {valid_chars_stringified}",
revision
)
.into());