mii/tokenizers
1
0
Fork 0
mirror of https://github.com/mii443/tokenizers.git synced 2025-12-08 21:58:18 +00:00
Code Issues Packages Projects Releases Wiki Activity

Black on pyi file.

Browse Source
This commit is contained in:
Nicolas Patry
2020-09-22 18:23:38 +02:00
parent 60c1e25910
commit 1cd4824273
1 changed files with 7 additions and 12 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
bindings/python/py_src/tokenizers/models/__init__.pyi
View File

@@ -62,6 +62,7 @@ class BPE(Model):
fuse_unk: (`optional`) bool:
Multiple unk tokens get fused into only 1
"""
def __init__(
self,
vocab: Optional[Union[str, Dict[str, int]]],
@@ -74,11 +75,9 @@ class BPE(Model):
fuse_unk: Optional[bool],
):
pass
@staticmethod
def read_files(vocab_filename: str, merges_filename: str) -> Tuple[Vocab, Merges]:
pass
@staticmethod
def from_files(vocab_filename: str, merges_filename: str, **kwargs) -> BPE:
vocab, merges = BPE.read_files(vocab_filename, merges_filename)
@@ -107,11 +106,9 @@ class WordPiece(Model):
max_input_chars_per_word: Optional[int],
):
pass
@staticmethod
def read_file(vocab_filename: str) -> Tuple[Vocab]:
pass
@staticmethod
def from_files(vocab_filename: str, **kwargs) -> WordPiece:
vocab = WordPiece.read_files(vocab_filename)
@@ -133,11 +130,9 @@ class WordLevel(Model):
def __init__(self, vocab: Optional[Union[str, Dict[str, int]]], unk_token: Optional[str]):
pass
@staticmethod
def read_file(vocab_filename: str) -> Tuple[Vocab]:
pass
@staticmethod
def from_files(vocab_filename: str, **kwargs) -> WordLevel:
vocab = WordLevel.read_files(vocab_filename)