From b17f9d887221b4fa1ba455a27e76faf55601c412 Mon Sep 17 00:00:00 2001 From: Anthony MOI Date: Tue, 7 Jan 2020 18:19:10 -0500 Subject: [PATCH] Rename ByteLevelBPE Rename ByteLevelBPETokenizer --- bindings/python/tokenizers/__init__.py | 2 +- bindings/python/tokenizers/__init__.pyi | 2 +- bindings/python/tokenizers/implementations/__init__.py | 2 +- bindings/python/tokenizers/implementations/byte_level_bpe.py | 4 ++-- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/bindings/python/tokenizers/__init__.py b/bindings/python/tokenizers/__init__.py index 8c94c6f4..00ec1a1b 100644 --- a/bindings/python/tokenizers/__init__.py +++ b/bindings/python/tokenizers/__init__.py @@ -7,4 +7,4 @@ from .tokenizers import normalizers from .tokenizers import pre_tokenizers from .tokenizers import processors from .tokenizers import trainers -from .implementations import ByteLevelBpe +from .implementations import ByteLevelBPETokenizer diff --git a/bindings/python/tokenizers/__init__.pyi b/bindings/python/tokenizers/__init__.pyi index 0dc97d57..e3b3cd00 100644 --- a/bindings/python/tokenizers/__init__.pyi +++ b/bindings/python/tokenizers/__init__.pyi @@ -5,7 +5,7 @@ from tokenizers import pre_tokenizers from tokenizers import processors from tokenizers import trainers -from tokenizers.implementations import ByteLevelBpe +from tokenizers.implementations import ByteLevelBPETokenizer from typing import Optional, Union, List, Tuple diff --git a/bindings/python/tokenizers/implementations/__init__.py b/bindings/python/tokenizers/implementations/__init__.py index 0ae2721d..a421cbcb 100644 --- a/bindings/python/tokenizers/implementations/__init__.py +++ b/bindings/python/tokenizers/implementations/__init__.py @@ -1,2 +1,2 @@ from .base_tokenizer import BaseTokenizer -from .byte_level_bpe import ByteLevelBpe +from .byte_level_bpe import ByteLevelBPETokenizer diff --git a/bindings/python/tokenizers/implementations/byte_level_bpe.py b/bindings/python/tokenizers/implementations/byte_level_bpe.py index 1e5655a4..dfe1f902 100644 --- a/bindings/python/tokenizers/implementations/byte_level_bpe.py +++ b/bindings/python/tokenizers/implementations/byte_level_bpe.py @@ -5,8 +5,8 @@ from .base_tokenizer import BaseTokenizer from typing import Optional -class ByteLevelBpe(BaseTokenizer): - """ ByteLevelBpe +class ByteLevelBPETokenizer(BaseTokenizer): + """ ByteLevelBPETokenizer Represents a Byte-level BPE as introduced by OpenAI with their GPT-2 model """