mii/tokenizers
1
0
Fork 0
mirror of https://github.com/mii443/tokenizers.git synced 2025-09-03 07:49:22 +00:00
Code Issues Packages Projects Releases Wiki Activity

Rename ByteLevelBPE

Browse Source 
Rename ByteLevelBPETokenizer
This commit is contained in:
Anthony MOI
2020-01-07 18:19:10 -05:00
parent 6d0e3ba8f1
commit b17f9d8872
4 changed files with 5 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
bindings/python/tokenizers/__init__.py
View File

@ -7,4 +7,4 @@ from .tokenizers import normalizers
from .tokenizers import pre_tokenizers from .tokenizers import pre_tokenizers
from .tokenizers import processors from .tokenizers import processors
from .tokenizers import trainers from .tokenizers import trainers
from .implementations import ByteLevelBpe from .implementations import ByteLevelBPETokenizer

2
bindings/python/tokenizers/__init__.pyi
View File

@ -5,7 +5,7 @@ from tokenizers import pre_tokenizers
from tokenizers import processors from tokenizers import processors
from tokenizers import trainers from tokenizers import trainers
from tokenizers.implementations import ByteLevelBpe from tokenizers.implementations import ByteLevelBPETokenizer
from typing import Optional, Union, List, Tuple from typing import Optional, Union, List, Tuple

2
bindings/python/tokenizers/implementations/__init__.py
View File

@ -1,2 +1,2 @@
from .base_tokenizer import BaseTokenizer from .base_tokenizer import BaseTokenizer
from .byte_level_bpe import ByteLevelBpe from .byte_level_bpe import ByteLevelBPETokenizer

4
bindings/python/tokenizers/implementations/byte_level_bpe.py
View File

@ -5,8 +5,8 @@ from .base_tokenizer import BaseTokenizer
from typing import Optional from typing import Optional
class ByteLevelBpe(BaseTokenizer): class ByteLevelBPETokenizer(BaseTokenizer):
""" ByteLevelBpe """ ByteLevelBPETokenizer
Represents a Byte-level BPE as introduced by OpenAI with their GPT-2 model Represents a Byte-level BPE as introduced by OpenAI with their GPT-2 model
""" """