diff --git a/docs/source-doc-builder/api/added-tokens.mdx b/docs/source-doc-builder/api/added-tokens.mdx index f5adc230..bd7a908c 100644 --- a/docs/source-doc-builder/api/added-tokens.mdx +++ b/docs/source-doc-builder/api/added-tokens.mdx @@ -2,7 +2,7 @@ -## AddedToken[[tokenizers.AddedToken]] +## AddedToken [[autodoc]] tokenizers.AddedToken - content diff --git a/docs/source-doc-builder/api/decoders.mdx b/docs/source-doc-builder/api/decoders.mdx index a3642305..44ee426d 100644 --- a/docs/source-doc-builder/api/decoders.mdx +++ b/docs/source-doc-builder/api/decoders.mdx @@ -2,23 +2,23 @@ -## BPEDecoder[[tokenizers.decoders.BPEDecoder]] +## BPEDecoder [[autodoc]] tokenizers.decoders.BPEDecoder -## ByteLevel[[tokenizers.decoders.ByteLevel]] +## ByteLevel [[autodoc]] tokenizers.decoders.ByteLevel -## CTC[[tokenizers.decoders.CTC]] +## CTC [[autodoc]] tokenizers.decoders.CTC -## Metaspace[[tokenizers.decoders.Metaspace]] +## Metaspace [[autodoc]] tokenizers.decoders.Metaspace -## WordPiece[[tokenizers.decoders.WordPiece]] +## WordPiece [[autodoc]] tokenizers.decoders.WordPiece diff --git a/docs/source-doc-builder/api/encoding.mdx b/docs/source-doc-builder/api/encoding.mdx index e8ab29e8..e88f2dea 100644 --- a/docs/source-doc-builder/api/encoding.mdx +++ b/docs/source-doc-builder/api/encoding.mdx @@ -2,7 +2,7 @@ -## Encoding[[tokenizers.Encoding]] +## Encoding [[autodoc]] tokenizers.Encoding - all diff --git a/docs/source-doc-builder/api/models.mdx b/docs/source-doc-builder/api/models.mdx index d19a9199..4795848f 100644 --- a/docs/source-doc-builder/api/models.mdx +++ b/docs/source-doc-builder/api/models.mdx @@ -2,23 +2,23 @@ -## BPE[[tokenizers.models.BPE]] +## BPE [[autodoc]] tokenizers.models.BPE -## Model[[tokenizers.models.Model]] +## Model [[autodoc]] tokenizers.models.Model -## Unigram[[tokenizers.models.Unigram]] +## Unigram [[autodoc]] tokenizers.models.Unigram -## WordLevel[[tokenizers.models.WordLevel]] +## WordLevel [[autodoc]] tokenizers.models.WordLevel -## WordPiece[[tokenizers.models.WordPiece]] +## WordPiece [[autodoc]] tokenizers.models.WordPiece diff --git a/docs/source-doc-builder/api/normalizers.mdx b/docs/source-doc-builder/api/normalizers.mdx index 8320911e..fc5186ac 100644 --- a/docs/source-doc-builder/api/normalizers.mdx +++ b/docs/source-doc-builder/api/normalizers.mdx @@ -2,55 +2,55 @@ -## BertNormalizer[[tokenizers.normalizers.BertNormalizer]] +## BertNormalizer [[autodoc]] tokenizers.normalizers.BertNormalizer -## Lowercase[[tokenizers.normalizers.Lowercase]] +## Lowercase [[autodoc]] tokenizers.normalizers.Lowercase -## NFC[[tokenizers.normalizers.NFC]] +## NFC [[autodoc]] tokenizers.normalizers.NFC -## NFD[[tokenizers.normalizers.NFD]] +## NFD [[autodoc]] tokenizers.normalizers.NFD -## NFKC[[tokenizers.normalizers.NFKC]] +## NFKC [[autodoc]] tokenizers.normalizers.NFKC -## NFKD[[tokenizers.normalizers.NFKD]] +## NFKD [[autodoc]] tokenizers.normalizers.NFKD -## Nmt[[tokenizers.normalizers.Nmt]] +## Nmt [[autodoc]] tokenizers.normalizers.Nmt -## Normalizer[[tokenizers.normalizers.Normalizer]] +## Normalizer [[autodoc]] tokenizers.normalizers.Normalizer -## Precompiled[[tokenizers.normalizers.Precompiled]] +## Precompiled [[autodoc]] tokenizers.normalizers.Precompiled -## Replace[[tokenizers.normalizers.Replace]] +## Replace [[autodoc]] tokenizers.normalizers.Replace -## Sequence[[tokenizers.normalizers.Sequence]] +## Sequence [[autodoc]] tokenizers.normalizers.Sequence -## Strip[[tokenizers.normalizers.Strip]] +## Strip [[autodoc]] tokenizers.normalizers.Strip -## StripAccents[[tokenizers.normalizers.StripAccents]] +## StripAccents [[autodoc]] tokenizers.normalizers.StripAccents diff --git a/docs/source-doc-builder/api/post-processors.mdx b/docs/source-doc-builder/api/post-processors.mdx index e55efeff..fdaad208 100644 --- a/docs/source-doc-builder/api/post-processors.mdx +++ b/docs/source-doc-builder/api/post-processors.mdx @@ -2,19 +2,19 @@ -## BertProcessing[[tokenizers.processors.BertProcessing]] +## BertProcessing [[autodoc]] tokenizers.processors.BertProcessing -## ByteLevel[[tokenizers.processors.ByteLevel]] +## ByteLevel [[autodoc]] tokenizers.processors.ByteLevel -## RobertaProcessing[[tokenizers.processors.RobertaProcessing]] +## RobertaProcessing [[autodoc]] tokenizers.processors.RobertaProcessing -## TemplateProcessing[[tokenizers.processors.TemplateProcessing]] +## TemplateProcessing [[autodoc]] tokenizers.processors.TemplateProcessing diff --git a/docs/source-doc-builder/api/pre-tokenizers.mdx b/docs/source-doc-builder/api/pre-tokenizers.mdx index 14d917f6..0f51565d 100644 --- a/docs/source-doc-builder/api/pre-tokenizers.mdx +++ b/docs/source-doc-builder/api/pre-tokenizers.mdx @@ -2,51 +2,51 @@ -## BertPreTokenizer[[tokenizers.pre_tokenizers.BertPreTokenizer]] +## BertPreTokenizer [[autodoc]] tokenizers.pre_tokenizers.BertPreTokenizer -## ByteLevel[[tokenizers.pre_tokenizers.ByteLevel]] +## ByteLevel [[autodoc]] tokenizers.pre_tokenizers.ByteLevel -## CharDelimiterSplit[[tokenizers.pre_tokenizers.CharDelimiterSplit]] +## CharDelimiterSplit [[autodoc]] tokenizers.pre_tokenizers.CharDelimiterSplit -## Digits[[tokenizers.pre_tokenizers.Digits]] +## Digits [[autodoc]] tokenizers.pre_tokenizers.Digits -## Metaspace[[tokenizers.pre_tokenizers.Metaspace]] +## Metaspace [[autodoc]] tokenizers.pre_tokenizers.Metaspace -## PreTokenizer[[tokenizers.pre_tokenizers.PreTokenizer]] +## PreTokenizer [[autodoc]] tokenizers.pre_tokenizers.PreTokenizer -## Punctuation[[tokenizers.pre_tokenizers.Punctuation]] +## Punctuation [[autodoc]] tokenizers.pre_tokenizers.Punctuation -## Sequence[[tokenizers.pre_tokenizers.Sequence]] +## Sequence [[autodoc]] tokenizers.pre_tokenizers.Sequence -## Split[[tokenizers.pre_tokenizers.Split]] +## Split [[autodoc]] tokenizers.pre_tokenizers.Split -## UnicodeScripts[[tokenizers.pre_tokenizers.UnicodeScripts]] +## UnicodeScripts [[autodoc]] tokenizers.pre_tokenizers.UnicodeScripts -## Whitespace[[tokenizers.pre_tokenizers.Whitespace]] +## Whitespace [[autodoc]] tokenizers.pre_tokenizers.Whitespace -## WhitespaceSplit[[tokenizers.pre_tokenizers.WhitespaceSplit]] +## WhitespaceSplit [[autodoc]] tokenizers.pre_tokenizers.WhitespaceSplit diff --git a/docs/source-doc-builder/api/tokenizer.mdx b/docs/source-doc-builder/api/tokenizer.mdx index ab351030..841f10f9 100644 --- a/docs/source-doc-builder/api/tokenizer.mdx +++ b/docs/source-doc-builder/api/tokenizer.mdx @@ -2,7 +2,7 @@ -## Tokenizer[[tokenizers.Tokenizer]] +## Tokenizer [[autodoc]] tokenizers.Tokenizer - all diff --git a/docs/source-doc-builder/api/trainers.mdx b/docs/source-doc-builder/api/trainers.mdx index cfaa976b..0e947494 100644 --- a/docs/source-doc-builder/api/trainers.mdx +++ b/docs/source-doc-builder/api/trainers.mdx @@ -2,19 +2,19 @@ -## BpeTrainer[[tokenizers.trainers.BpeTrainer]] +## BpeTrainer [[autodoc]] tokenizers.trainers.BpeTrainer -## UnigramTrainer[[tokenizers.trainers.UnigramTrainer]] +## UnigramTrainer [[autodoc]] tokenizers.trainers.UnigramTrainer -## WordLevelTrainer[[tokenizers.trainers.WordLevelTrainer]] +## WordLevelTrainer [[autodoc]] tokenizers.trainers.WordLevelTrainer -## WordPieceTrainer[[tokenizers.trainers.WordPieceTrainer]] +## WordPieceTrainer [[autodoc]] tokenizers.trainers.WordPieceTrainer diff --git a/docs/source-doc-builder/api/visualizer.mdx b/docs/source-doc-builder/api/visualizer.mdx index 5e7e4d88..0a7aca8a 100644 --- a/docs/source-doc-builder/api/visualizer.mdx +++ b/docs/source-doc-builder/api/visualizer.mdx @@ -2,11 +2,11 @@ -## Annotation[[tokenizers.tools.Annotation]] +## Annotation [[autodoc]] tokenizers.tools.Annotation -## EncodingVisualizer[[tokenizers.tools.EncodingVisualizer]] +## EncodingVisualizer [[autodoc]] tokenizers.tools.EncodingVisualizer - __call__