mii/tokenizers
1
0
Fork 0
mirror of https://github.com/mii443/tokenizers.git synced 2025-08-22 16:25:30 +00:00
Code Issues Packages Projects Releases Wiki Activity

import Tuple from typing (#1295)

Browse Source
This commit is contained in:
Kelly Marchisio
2023-07-14 11:39:29 -04:00
committed by GitHub
parent 150559b61e
commit 4811f769a1
1 changed files with 1 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
bindings/python/py_src/tokenizers/implementations/sentencepiece_unigram.py
View File

@ -1,6 +1,6 @@
import json import json
import os import os
from typing import Iterator, List, Optional, Union from typing import Iterator, List, Optional, Union, Tuple
from tokenizers import AddedToken, Regex, Tokenizer, decoders, normalizers, pre_tokenizers, trainers from tokenizers import AddedToken, Regex, Tokenizer, decoders, normalizers, pre_tokenizers, trainers
from tokenizers.models import Unigram from tokenizers.models import Unigram