mirror of
https://github.com/mii443/tokenizers.git
synced 2025-08-22 16:25:30 +00:00
63 lines
1.4 KiB
Plaintext
63 lines
1.4 KiB
Plaintext
# Normalizers
|
|
|
|
<tokenizerslangcontent>
|
|
<python>
|
|
## BertNormalizer[[tokenizers.normalizers.BertNormalizer]]
|
|
|
|
[[autodoc]] tokenizers.normalizers.BertNormalizer
|
|
|
|
## Lowercase[[tokenizers.normalizers.Lowercase]]
|
|
|
|
[[autodoc]] tokenizers.normalizers.Lowercase
|
|
|
|
## NFC[[tokenizers.normalizers.NFC]]
|
|
|
|
[[autodoc]] tokenizers.normalizers.NFC
|
|
|
|
## NFD[[tokenizers.normalizers.NFD]]
|
|
|
|
[[autodoc]] tokenizers.normalizers.NFD
|
|
|
|
## NFKC[[tokenizers.normalizers.NFKC]]
|
|
|
|
[[autodoc]] tokenizers.normalizers.NFKC
|
|
|
|
## NFKD[[tokenizers.normalizers.NFKD]]
|
|
|
|
[[autodoc]] tokenizers.normalizers.NFKD
|
|
|
|
## Nmt[[tokenizers.normalizers.Nmt]]
|
|
|
|
[[autodoc]] tokenizers.normalizers.Nmt
|
|
|
|
## Normalizer[[tokenizers.normalizers.Normalizer]]
|
|
|
|
[[autodoc]] tokenizers.normalizers.Normalizer
|
|
|
|
## Precompiled[[tokenizers.normalizers.Precompiled]]
|
|
|
|
[[autodoc]] tokenizers.normalizers.Precompiled
|
|
|
|
## Replace[[tokenizers.normalizers.Replace]]
|
|
|
|
[[autodoc]] tokenizers.normalizers.Replace
|
|
|
|
## Sequence[[tokenizers.normalizers.Sequence]]
|
|
|
|
[[autodoc]] tokenizers.normalizers.Sequence
|
|
|
|
## Strip[[tokenizers.normalizers.Strip]]
|
|
|
|
[[autodoc]] tokenizers.normalizers.Strip
|
|
|
|
## StripAccents[[tokenizers.normalizers.StripAccents]]
|
|
|
|
[[autodoc]] tokenizers.normalizers.StripAccents
|
|
</python>
|
|
<rust>
|
|
The Rust API Reference is available directly on the [Docs.rs](https://docs.rs/tokenizers/latest/tokenizers/) website.
|
|
</rust>
|
|
<node>
|
|
The node API has not been documented yet.
|
|
</node>
|
|
</tokenizerslangcontent> |