mii/tokenizers
1
0
Fork 0
mirror of https://github.com/mii443/tokenizers.git synced 2025-08-23 00:35:35 +00:00
Code Issues Packages Projects Releases Wiki Activity

Remove some useless modifications.

Browse Source
This commit is contained in:
Nicolas Patry
2020-09-18 13:48:35 +02:00
parent c536b4992b
commit 8e220dbdd4
2 changed files with 0 additions and 10 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
tokenizers/Cargo.toml
View File

@ -58,4 +58,3 @@ spm_precompiled = "0.1"
criterion = "0.3" criterion = "0.3"
tempfile = "3.1" tempfile = "3.1"
assert_approx_eq = "1.1" assert_approx_eq = "1.1"
simple_logger = "1.9"

9
tokenizers/src/tokenizer/normalizer.rs
View File

@ -489,15 +489,6 @@ impl NormalizedString {
} }
}) })
.collect::<Vec<_>>(); .collect::<Vec<_>>();
debug!(
"Filter transform {:?}",
filtered
.iter()
.cloned()
.rev()
.filter_map(|o| o)
.collect::<Vec<_>>()
);
self.transform( self.transform(
filtered.into_iter().rev().filter_map(|o| o), filtered.into_iter().rev().filter_map(|o| o),
removed as usize, removed as usize,