mirror of
https://github.com/mii443/tokenizers.git
synced 2025-08-22 16:25:30 +00:00
63 lines
1003 B
Plaintext
63 lines
1003 B
Plaintext
# Normalizers
|
|
|
|
<tokenizerslangcontent>
|
|
<python>
|
|
## BertNormalizer
|
|
|
|
[[autodoc]] tokenizers.normalizers.BertNormalizer
|
|
|
|
## Lowercase
|
|
|
|
[[autodoc]] tokenizers.normalizers.Lowercase
|
|
|
|
## NFC
|
|
|
|
[[autodoc]] tokenizers.normalizers.NFC
|
|
|
|
## NFD
|
|
|
|
[[autodoc]] tokenizers.normalizers.NFD
|
|
|
|
## NFKC
|
|
|
|
[[autodoc]] tokenizers.normalizers.NFKC
|
|
|
|
## NFKD
|
|
|
|
[[autodoc]] tokenizers.normalizers.NFKD
|
|
|
|
## Nmt
|
|
|
|
[[autodoc]] tokenizers.normalizers.Nmt
|
|
|
|
## Normalizer
|
|
|
|
[[autodoc]] tokenizers.normalizers.Normalizer
|
|
|
|
## Precompiled
|
|
|
|
[[autodoc]] tokenizers.normalizers.Precompiled
|
|
|
|
## Replace
|
|
|
|
[[autodoc]] tokenizers.normalizers.Replace
|
|
|
|
## Sequence
|
|
|
|
[[autodoc]] tokenizers.normalizers.Sequence
|
|
|
|
## Strip
|
|
|
|
[[autodoc]] tokenizers.normalizers.Strip
|
|
|
|
## StripAccents
|
|
|
|
[[autodoc]] tokenizers.normalizers.StripAccents
|
|
</python>
|
|
<rust>
|
|
The Rust API Reference is available directly on the [Docs.rs](https://docs.rs/tokenizers/latest/tokenizers/) website.
|
|
</rust>
|
|
<node>
|
|
The node API has not been documented yet.
|
|
</node>
|
|
</tokenizerslangcontent> |