mirror of
https://github.com/mii443/tokenizers.git
synced 2025-08-23 00:35:35 +00:00
Remove Tokenizer::normalize
This is actually a legacy function that doesn't really make sense now, and is getting really difficult to keep. So we remove it.
This commit is contained in:
@ -509,15 +509,6 @@ impl PyTokenizer {
|
||||
})
|
||||
}
|
||||
|
||||
fn normalize(&self, sentence: &str) -> PyResult<String> {
|
||||
ToPyResult(
|
||||
self.tokenizer
|
||||
.normalize(sentence)
|
||||
.map(|s| s.get().to_owned()),
|
||||
)
|
||||
.into()
|
||||
}
|
||||
|
||||
/// Input can be:
|
||||
/// encode("A single sequence")
|
||||
/// encode("A sequence", "And its pair")
|
||||
|
Reference in New Issue
Block a user