mii/tokenizers
1
0
Fork 0
mirror of https://github.com/mii443/tokenizers.git synced 2025-09-03 15:59:25 +00:00
Code Issues Packages Projects Releases Wiki Activity

Rename ByteLevelBPE

Browse Source 
Rename ByteLevelBPETokenizer
This commit is contained in:
Anthony MOI
2020-01-07 18:19:10 -05:00
parent 6d0e3ba8f1
commit b17f9d8872
4 changed files with 5 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
bindings/python/tokenizers/__init__.py
View File

@ -7,4 +7,4 @@ from .tokenizers import normalizers
from .tokenizers import pre_tokenizers
from .tokenizers import processors
from .tokenizers import trainers
from .implementations import ByteLevelBpe
from .implementations import ByteLevelBPETokenizer

2
bindings/python/tokenizers/__init__.pyi
View File

@ -5,7 +5,7 @@ from tokenizers import pre_tokenizers
from tokenizers import processors
from tokenizers import trainers
from tokenizers.implementations import ByteLevelBpe
from tokenizers.implementations import ByteLevelBPETokenizer
from typing import Optional, Union, List, Tuple

2
bindings/python/tokenizers/implementations/__init__.py
View File

@ -1,2 +1,2 @@
from .base_tokenizer import BaseTokenizer
from .byte_level_bpe import ByteLevelBpe
from .byte_level_bpe import ByteLevelBPETokenizer

4
bindings/python/tokenizers/implementations/byte_level_bpe.py
View File

@ -5,8 +5,8 @@ from .base_tokenizer import BaseTokenizer
from typing import Optional
class ByteLevelBpe(BaseTokenizer):
""" ByteLevelBpe
class ByteLevelBPETokenizer(BaseTokenizer):
""" ByteLevelBPETokenizer
Represents a Byte-level BPE as introduced by OpenAI with their GPT-2 model
"""