mii/tokenizers
1
0
Fork 0
mirror of https://github.com/mii443/tokenizers.git synced 2025-12-07 13:18:31 +00:00
Code Issues Packages Projects Releases Wiki Activity

Added missing imports for AddedToken

Browse Source
This commit is contained in:
Morgan Funtowicz
2020-03-27 12:32:23 +01:00
parent b132be34af
commit bc46064a61
1 changed files with 1 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
bindings/python/tokenizers/implementations/base_tokenizer.py
View File

@@ -1,4 +1,4 @@
from tokenizers import Tokenizer, Encoding from tokenizers import Tokenizer, Encoding, AddedToken
from tokenizers.models import TokenizedSequence, TokenizedSequenceWithOffsets from tokenizers.models import TokenizedSequence, TokenizedSequenceWithOffsets
from typing import List, Union, Tuple, Optional from typing import List, Union, Tuple, Optional