diff --git a/.gitignore b/.gitignore index 73fe5274..20819124 100644 --- a/.gitignore +++ b/.gitignore @@ -15,3 +15,5 @@ pip-wheel-metadata *.so /bindings/python/build /bindings/python/dist + +.vscode \ No newline at end of file diff --git a/bindings/python/tokenizers/implementations/base_tokenizer.py b/bindings/python/tokenizers/implementations/base_tokenizer.py index 8a7d8854..e34bb075 100644 --- a/bindings/python/tokenizers/implementations/base_tokenizer.py +++ b/bindings/python/tokenizers/implementations/base_tokenizer.py @@ -1,6 +1,6 @@ -from .tokenizers import Tokenizer +from ..tokenizers import Tokenizer, Encoding -from typing import List, Union, Tuple +from typing import List, Union, Tuple, Optional class BaseTokenizer: _tokenizer: Tokenizer diff --git a/bindings/python/tokenizers/implementations/byte_level_bpe.py b/bindings/python/tokenizers/implementations/byte_level_bpe.py index aceee62e..1e5655a4 100644 --- a/bindings/python/tokenizers/implementations/byte_level_bpe.py +++ b/bindings/python/tokenizers/implementations/byte_level_bpe.py @@ -1,7 +1,7 @@ from tokenizers import Tokenizer, pre_tokenizers, decoders -from tokenizers.tokenizers import BaseTokenizer from tokenizers.models import BPE from tokenizers.normalizers import NFKC +from .base_tokenizer import BaseTokenizer from typing import Optional @@ -14,7 +14,7 @@ class ByteLevelBpe(BaseTokenizer): def __init__(self, vocab_file: Optional[str]=None, merges_file: Optional[str]=None, - add_prefix_space: boolean=False): + add_prefix_space: bool=False): if vocab_file is not None and merges_file is not None: tokenizer = Tokenizer(BPE.from_files(vocab_file, merges_file)) else: