mii/tokenizers
1
0
Fork 0
mirror of https://github.com/mii443/tokenizers.git synced 2025-08-22 16:25:30 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix imports

Browse Source
This commit is contained in:
thomwolf
2020-01-07 22:55:50 +01:00
committed by Anthony MOI
parent 63063118df
commit 6d0e3ba8f1
3 changed files with 6 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
.gitignore vendored
View File

@ -15,3 +15,5 @@ pip-wheel-metadata
*.so *.so
/bindings/python/build /bindings/python/build
/bindings/python/dist /bindings/python/dist
.vscode

4
bindings/python/tokenizers/implementations/base_tokenizer.py
View File

@ -1,6 +1,6 @@
from .tokenizers import Tokenizer from ..tokenizers import Tokenizer, Encoding
from typing import List, Union, Tuple from typing import List, Union, Tuple, Optional
class BaseTokenizer: class BaseTokenizer:
_tokenizer: Tokenizer _tokenizer: Tokenizer

4
bindings/python/tokenizers/implementations/byte_level_bpe.py
View File

@ -1,7 +1,7 @@
from tokenizers import Tokenizer, pre_tokenizers, decoders from tokenizers import Tokenizer, pre_tokenizers, decoders
from tokenizers.tokenizers import BaseTokenizer
from tokenizers.models import BPE from tokenizers.models import BPE
from tokenizers.normalizers import NFKC from tokenizers.normalizers import NFKC
from .base_tokenizer import BaseTokenizer
from typing import Optional from typing import Optional
@ -14,7 +14,7 @@ class ByteLevelBpe(BaseTokenizer):
def __init__(self, def __init__(self,
vocab_file: Optional[str]=None, vocab_file: Optional[str]=None,
merges_file: Optional[str]=None, merges_file: Optional[str]=None,
add_prefix_space: boolean=False): add_prefix_space: bool=False):
if vocab_file is not None and merges_file is not None: if vocab_file is not None and merges_file is not None:
tokenizer = Tokenizer(BPE.from_files(vocab_file, merges_file)) tokenizer = Tokenizer(BPE.from_files(vocab_file, merges_file))
else: else: