Python - Adding some stub files

bindings/python/tokenizers/decoders/__init__.pyi

@@ -0,0 +1,28 @@
+from .. import decoders
+
+class Decoder:
+    """Decoder
+    """
+
+    @staticmethod
+    def custom():
+        pass
+
+    def decode(tokens: List[str]) -> str:
+        pass
+
+class ByteLevel:
+    """ByteLevel
+    """
+
+    @staticmethod
+    def new() -> Decoder:
+        pass
+
+class WordPiece:
+    """WordPiece
+    """
+
+    @staticmethod
+    def new() -> Decoder:
+        pass

bindings/python/tokenizers/models/__init__.pyi

@@ -0,0 +1,33 @@
+from .. import models
+
+class Model:
+    """Model
+    """
+
+    def save(folder: str, name: str) -> List[str]:
+        """ save
+        Save the current Model in the given folder, using the given name for the various
+        files that will get created.
+        Any file with the same name that already exist in this folder will be overwritten
+        """
+        pass
+
+class BPE:
+    """BPE
+    """
+
+    def from_files(vocab: str, merges: str) -> Model:
+        pass
+
+    def empty() -> Model:
+        pass
+
+class WordPiece:
+    """WordPiece
+    """
+
+    def from_files(vocab: str) -> Model:
+        pass
+
+    def empty() -> Model:
+        pass

bindings/python/tokenizers/normalizers/__init__.pyi

@@ -0,0 +1,12 @@
+from .. import normalizers
+
+class Normalizer:
+    """Normalizer
+    """
+
+class BertNormalizer:
+    """BertNormalizer
+    """
+
+    def new() -> Normalizer:
+        pass

bindings/python/tokenizers/pre_tokenizers/__init__.pyi

@@ -0,0 +1,38 @@
+from .. import pre_tokenizers
+
+Offsets = Tuple[int, int]
+
+class PreTokenizer:
+    """PreTokenizer
+    """
+
+    def pre_tokenize(self, sequence: str) -> List[Tuple[str, Offsets]]:
+        pass
+
+class ByteLevel:
+    """ByteLevel
+    """
+
+    @staticmethod
+    def new() -> PreTokenizer:
+        pass
+
+    @staticmethod
+    def alphabet() -> List[str]:
+        pass
+
+class Whitespace:
+    """Whitespace
+    """
+
+    @staticmethod
+    def new() -> PreTokenizer:
+        pass
+
+class BertPreTokenizer:
+    """BertPreTokenizer
+    """
+
+    @staticmethod
+    def new() -> PreTokenizer:
+        pass

bindings/python/tokenizers/processors/__init__.pyi

@@ -0,0 +1,13 @@
+from .. import processors
+
+class PostProcessor:
+    """PostProcessor
+    """
+
+class BertProcessing:
+    """BertProcessing
+    """
+
+    @staticmethod
+    def new(sep: Tuple[str, int], cls: Tuple[str, int]) -> PostProcessor:
+        pass

bindings/python/tokenizers/trainers/__init__.pyi

@@ -0,0 +1,21 @@
+from .. import trainers
+
+class Trainer:
+    """Trainer
+    """
+
+class BpeTrainer:
+    """BpeTrainer
+    """
+
+    @staticmethod
+    def new() -> Trainer:
+        pass
+
+class WordPieceTrainer:
+    """WordPieceTrainer
+    """
+
+    @staticmethod
+    def new() -> Trainer:
+        pass