mii/tokenizers
1
0
Fork 0
mirror of https://github.com/mii443/tokenizers.git synced 2025-12-08 13:48:19 +00:00
Code Issues Packages Projects Releases Wiki Activity

Python - Add bindings to Tokenizer.from_pretrained

Browse Source
This commit is contained in:
Anthony Moi
2021-08-19 11:45:23 +02:00
committed by Anthony MOI
parent e71e5be64f
commit e44fdee4a1
4 changed files with 1022 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

20
bindings/python/py_src/tokenizers/__init__.pyi
View File

@@ -858,6 +858,26 @@ class Tokenizer:
"""
pass
@staticmethod
def from_pretrained(identifier, revision="main", auth_token=None):
"""
Instantiate a new :class:`~tokenizers.Tokenizer` from an existing file on the
Hugging Face Hub.
Args:
identifier (:obj:`str`):
The identifier of a Model on the Hugging Face Hub, that contains
a tokenizer.json file
revision (:obj:`str`, defaults to `main`):
A branch or commit id
auth_token (:obj:`str`, `optional`, defaults to `None`):
An optional auth token used to access private repositories on the
Hugging Face Hub
Returns:
:class:`~tokenizers.Tokenizer`: The new tokenizer
"""
pass
@staticmethod
def from_str(json):
"""
Instantiate a new :class:`~tokenizers.Tokenizer` from the given JSON string.