From a7ace4480d1cc6404e6ba7567c543fb5ca2aa2b0 Mon Sep 17 00:00:00 2001 From: Arthur Zucker Date: Tue, 5 Sep 2023 17:33:14 +0000 Subject: [PATCH] `python stub.py ` --- bindings/python/py_src/tokenizers/__init__.pyi | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/bindings/python/py_src/tokenizers/__init__.pyi b/bindings/python/py_src/tokenizers/__init__.pyi index b73c838c..4b80a7f7 100644 --- a/bindings/python/py_src/tokenizers/__init__.pyi +++ b/bindings/python/py_src/tokenizers/__init__.pyi @@ -899,6 +899,14 @@ class Tokenizer: :class:`~tokenizers.Tokenizer`: The new tokenizer """ pass + def get_added_tokens_decoder(self): + """ + Get the underlying vocabulary + + Returns: + :obj:`Dict[int, AddedToken]`: The vocabulary + """ + pass def get_vocab(self, with_added_tokens=True): """ Get the underlying vocabulary