Doc - Add some roles for Rust doc

docs/source/_ext/rust_doc.py

@@ -0,0 +1,81 @@
+from docutils import nodes
+
+import sphinx
+from sphinx.locale import _
+
+from conf import rust_version
+
+logger = sphinx.util.logging.getLogger(__name__)
+
+
+class RustRef:
+    def __call__(self, name, rawtext, text, lineno, inliner, options={}, content=[]):
+        doctype = name.split(":")[1]
+        parts = text.split("::")
+
+        if text.startswith("~"):
+            content = parts[-1]
+            parts[0] = parts[0][1:]
+        else:
+            content = text
+        link = self.base_link()
+
+        if doctype == "struct":
+            link += self.make_struct_link(parts)
+        if doctype == "func":
+            link += self.make_func_link(parts)
+        if doctype == "meth":
+            link += self.make_meth_link(parts)
+
+        node = nodes.reference(internal=False, refuri=link, text=content)
+        wrapper = nodes.literal(classes=["xref"])
+        wrapper += node
+
+        return [wrapper], []
+
+    def base_link(self):
+        return f"https://docs.rs/tokenizers/{rust_version}"
+
+    def make_struct_link(self, parts):
+        link = ""
+        struct_name = parts[-1]
+        path = parts[:-1]
+
+        for p in path:
+            link += f"/{p}"
+        link += f"/struct.{struct_name}.html"
+
+        return link
+
+    def make_func_link(self, parts):
+        link = ""
+        fn_name = parts[-1]
+
+        path = parts[:-1]
+        for p in path:
+            link += f"/{p}"
+        link += f"/fn.{fn_name}.html"
+
+        return link
+
+    def make_meth_link(self, parts):
+        meth_name = parts[-1]
+        if meth_name.endswith("()"):
+            meth_name = meth_name[:-2]
+
+        link = self.make_struct_link(parts[:-1])
+        link += f"#method.{meth_name}"
+
+        return link
+
+
+def setup(app):
+    app.add_role("rust:struct", RustRef())
+    app.add_role("rust:func", RustRef())
+    app.add_role("rust:meth", RustRef())
+
+    return {
+        "version": "0.1",
+        "parallel_read_safe": True,
+        "parallel_write_safe": True,
+    }

docs/source/conf.py

@@ -23,18 +23,23 @@ project = "tokenizers"
 copyright = "2020, huggingface"
 author = "huggingface"
 
-languages = ["node", "rust", "python"]
-
 # The full version, including alpha/beta/rc tags
 release = "0.9.0"
 
+# -- Custom information ------------------------------------------------------
+
+# The possible values for languages (used by `_ext/entities`)
+languages = ["node", "rust", "python"]
+
+# This defines the version used to generate links to docs.rs
+rust_version = "latest"
 
 # -- General configuration ---------------------------------------------------
 
 # Add any Sphinx extension module names here, as strings. They can be
 # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
 # ones.
-extensions = ["sphinx.ext.autodoc", "sphinx.ext.napoleon", "entities"]
+extensions = ["sphinx.ext.autodoc", "sphinx.ext.napoleon", "entities", "rust_doc"]
 
 # Add any paths that contain templates here, relative to this directory.
 templates_path = ["_templates"]