diff --git a/bindings/python/tokenizers/implementations/base_tokenizer.py b/bindings/python/tokenizers/implementations/base_tokenizer.py index e6a8a5cc..a33693c6 100644 --- a/bindings/python/tokenizers/implementations/base_tokenizer.py +++ b/bindings/python/tokenizers/implementations/base_tokenizer.py @@ -1,4 +1,4 @@ -from tokenizers import Tokenizer, Encoding +from tokenizers import Tokenizer, Encoding, AddedToken from tokenizers.models import TokenizedSequence, TokenizedSequenceWithOffsets from typing import List, Union, Tuple, Optional