From cace6561d74024899d119cd096e79477bc5fbe20 Mon Sep 17 00:00:00 2001 From: Anthony MOI Date: Tue, 27 Oct 2020 17:29:36 -0400 Subject: [PATCH] Doc - Extract global entities to their own file --- docs/source/entities.inc | 98 ++++++++++++++++++++++++++++++++++++++++ docs/source/index.rst | 59 +----------------------- 2 files changed, 99 insertions(+), 58 deletions(-) create mode 100644 docs/source/entities.inc diff --git a/docs/source/entities.inc b/docs/source/entities.inc new file mode 100644 index 00000000..59390a46 --- /dev/null +++ b/docs/source/entities.inc @@ -0,0 +1,98 @@ +.. entities:: python + + :global: + + class + class + classmethod + class method + Tokenizer + :class:`~tokenizers.Tokenizer` + Tokenizer.train + :meth:`~tokenizers.Tokenizer.train` + Tokenizer.save + :meth:`~tokenizers.Tokenizer.save` + Tokenizer.from_file + :meth:`~tokenizers.Tokenizer.from_file` + Tokenizer.encode + :meth:`~tokenizers.Tokenizer.encode` + Tokenizer.encode_batch + :meth:`~tokenizers.Tokenizer.encode_batch` + Tokenizer.token_to_id + :meth:`~tokenizers.Tokenizer.token_to_id` + Tokenizer.enable_padding + :meth:`~tokenizers.Tokenizer.enable_padding` + Encoding + :class:`~tokenizers.Encoding` + TemplateProcessing + :class:`~tokenizers.processors.TemplateProcessing` + Normalizer + :class:`~tokenizers.normalizers.Normalizer` + normalizers.Sequence + :class:`~tokenizers.normalizers.Sequence` + +.. entities:: rust + + :global: + + class + struct + classmethod + static method + Tokenizer + :rust:struct:`~tokenizers::tokenizer::Tokenizer` + Tokenizer.train + :rust:meth:`~tokenizers::tokenizer::Tokenizer::train` + Tokenizer.save + :rust:meth:`~tokenizers::tokenizer::Tokenizer::save` + Tokenizer.from_file + :rust:meth:`~tokenizers::tokenizer::Tokenizer::from_file` + Tokenizer.encode + :rust:meth:`~tokenizers::tokenizer::Tokenizer::encode` + Tokenizer.encode_batch + :rust:meth:`~tokenizers::tokenizer::Tokenizer::encode_batch` + Tokenizer.token_to_id + :rust:meth:`~tokenizers::tokenizer::Tokenizer::token_to_id` + Tokenizer.enable_padding + :rust:meth:`~tokenizers::tokenizer::Tokenizer::enable_padding` + Encoding + :rust:struct:`~tokenizers::tokenizer::Encoding` + TemplateProcessing + :rust:struct:`~tokenizers::processors::template::TemplateProcessing` + Normalizer + :rust:trait:`~tokenizers::tokenizer::Normalizer` + normalizers.Sequence + :rust:struct:`~tokenizers::normalizers::utils::Sequence` + +.. entities:: node + + :global: + + class + class + classmethod + static method + Tokenizer + :obj:`Tokenizer` + Tokenizer.train + :obj:`Tokenizer.train()` + Tokenizer.save + :obj:`Tokenizer.save()` + Tokenizer.from_file + :obj:`Tokenizer.fromFile()` + Tokenizer.encode + :obj:`Tokenizer.encode()` + Tokenizer.encode_batch + :obj:`Tokenizer.encodeBatch()` + Tokenizer.token_to_id + :obj:`Tokenizer.tokenToId()` + Tokenizer.enable_padding + :obj:`Tokenizer.setPadding()` + Encoding + :obj:`Encoding` + TemplateProcessing + :obj:`TemplateProcessing` + Normalizer + :obj:`Normalizer` + normalizers.Sequence + :obj:`Sequence` diff --git a/docs/source/index.rst b/docs/source/index.rst index 19f24e8b..0e988f4d 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -38,61 +38,4 @@ Main features: api/reference - -.. entities:: python - - :global: - - class - class - classmethod - class method - Tokenizer - :class:`~tokenizers.Tokenizer` - Tokenizer.train - :meth:`~tokenizers.Tokenizer.train` - Tokenizer.save - :meth:`~tokenizers.Tokenizer.save` - Tokenizer.from_file - :meth:`~tokenizers.Tokenizer.from_file` - Tokenizer.encode - :meth:`~tokenizers.Tokenizer.encode` - Tokenizer.encode_batch - :meth:`~tokenizers.Tokenizer.encode_batch` - Tokenizer.token_to_id - :meth:`~tokenizers.Tokenizer.token_to_id` - Tokenizer.enable_padding - :meth:`~tokenizers.Tokenizer.enable_padding` - Encoding - :class:`~tokenizers.Encoding` - TemplateProcessing - :class:`~tokenizers.processors.TemplateProcessing` - -.. entities:: rust - - :global: - - class - struct - classmethod - static method - Tokenizer - :rust:struct:`~tokenizers::tokenizer::Tokenizer` - Tokenizer.train - :rust:meth:`~tokenizers::tokenizer::Tokenizer::train` - Tokenizer.save - :rust:meth:`~tokenizers::tokenizer::Tokenizer::save` - Tokenizer.from_file - :rust:meth:`~tokenizers::tokenizer::Tokenizer::from_file` - Tokenizer.encode - :rust:meth:`~tokenizers::tokenizer::Tokenizer::encode` - Tokenizer.encode_batch - :rust:meth:`~tokenizers::tokenizer::Tokenizer::encode_batch` - Tokenizer.token_to_id - :rust:meth:`~tokenizers::tokenizer::Tokenizer::token_to_id` - Tokenizer.enable_padding - :rust:meth:`~tokenizers::tokenizer::Tokenizer::enable_padding` - Encoding - :rust:struct:`~tokenizers::tokenizer::Encoding` - TemplateProcessing - :rust:struct:`~tokenizers::processors::template::TemplateProcessing` +.. include:: entities.inc