mii/tokenizers
1
0
Fork 0
mirror of https://github.com/mii443/tokenizers.git synced 2025-12-08 21:58:18 +00:00
Code Issues Packages Projects Releases Wiki Activity

Black on pyi file.

Browse Source
This commit is contained in:
Nicolas Patry
2020-09-22 18:23:38 +02:00
parent 60c1e25910
commit 1cd4824273
1 changed files with 7 additions and 12 deletions
Download Patch File Download Diff File Expand all files Collapse all files

19
bindings/python/py_src/tokenizers/models/__init__.pyi
View File

@@ -2,7 +2,7 @@ from .. import Encoding, Offsets, Token
from typing import List, Optional, Union, Tuple, Dict from typing import List, Optional, Union, Tuple, Dict
class Model: class Model:
""" Base class for all models """Base class for all models
This class is not supposed to be instantiated directly. Instead, any implementation of This class is not supposed to be instantiated directly. Instead, any implementation of
a Model will return a instance of this class when instantiated. a Model will return a instance of this class when instantiated.
@@ -18,7 +18,7 @@ class Model:
""" Returns the token associated with the given id """ """ Returns the token associated with the given id """
pass pass
def save(self, folder: str, name: Optional[str] = None) -> List[str]: def save(self, folder: str, name: Optional[str] = None) -> List[str]:
""" Save the current model """Save the current model
Save the current model in the given folder, using the given name for the various Save the current model in the given folder, using the given name for the various
files that will get created. files that will get created.
@@ -62,6 +62,7 @@ class BPE(Model):
fuse_unk: (`optional`) bool: fuse_unk: (`optional`) bool:
Multiple unk tokens get fused into only 1 Multiple unk tokens get fused into only 1
""" """
def __init__( def __init__(
self, self,
vocab: Optional[Union[str, Dict[str, int]]], vocab: Optional[Union[str, Dict[str, int]]],
@@ -74,18 +75,16 @@ class BPE(Model):
fuse_unk: Optional[bool], fuse_unk: Optional[bool],
): ):
pass pass
@staticmethod @staticmethod
def read_files(vocab_filename: str, merges_filename: str) -> Tuple[Vocab, Merges]: def read_files(vocab_filename: str, merges_filename: str) -> Tuple[Vocab, Merges]:
pass pass
@staticmethod @staticmethod
def from_files(vocab_filename: str, merges_filename: str, **kwargs) -> BPE: def from_files(vocab_filename: str, merges_filename: str, **kwargs) -> BPE:
vocab, merges = BPE.read_files(vocab_filename, merges_filename) vocab, merges = BPE.read_files(vocab_filename, merges_filename)
return BPE(vocab, merges, **kwargs) return BPE(vocab, merges, **kwargs)
class WordPiece(Model): class WordPiece(Model):
""" WordPiece model class """WordPiece model class
Instantiate a WordPiece Model from the given vocab file. Instantiate a WordPiece Model from the given vocab file.
@@ -107,14 +106,12 @@ class WordPiece(Model):
max_input_chars_per_word: Optional[int], max_input_chars_per_word: Optional[int],
): ):
pass pass
@staticmethod @staticmethod
def read_file(vocab_filename: str) -> Tuple[Vocab]: def read_file(vocab_filename: str) -> Tuple[Vocab]:
pass pass
@staticmethod @staticmethod
def from_files(vocab_filename: str, **kwargs) -> WordPiece: def from_files(vocab_filename: str, **kwargs) -> WordPiece:
vocab = WordPiece.read_files(vocab_filename) vocab = WordPiece.read_files(vocab_filename)
return WordPiece(vocab, **kwargs) return WordPiece(vocab, **kwargs)
class WordLevel(Model): class WordLevel(Model):
@@ -133,14 +130,12 @@ class WordLevel(Model):
def __init__(self, vocab: Optional[Union[str, Dict[str, int]]], unk_token: Optional[str]): def __init__(self, vocab: Optional[Union[str, Dict[str, int]]], unk_token: Optional[str]):
pass pass
@staticmethod @staticmethod
def read_file(vocab_filename: str) -> Tuple[Vocab]: def read_file(vocab_filename: str) -> Tuple[Vocab]:
pass pass
@staticmethod @staticmethod
def from_files(vocab_filename: str, **kwargs) -> WordLevel: def from_files(vocab_filename: str, **kwargs) -> WordLevel:
vocab = WordLevel.read_files(vocab_filename) vocab = WordLevel.read_files(vocab_filename)
return WordLevel(vocab, **kwargs) return WordLevel(vocab, **kwargs)
class Unigram(Model): class Unigram(Model):