mii/tokenizers
1
0
Fork 0
mirror of https://github.com/mii443/tokenizers.git synced 2025-12-03 11:18:29 +00:00
Code Issues Packages Projects Releases Wiki Activity

Fixed BPE.read_files -> BPE.read_file in SentencePieceBPETokenizer

Browse Source
This commit is contained in:
Timur Ganiev
2020-10-23 17:30:38 +03:00
committed by Anthony MOI
parent a2289d49b4
commit f7c61c267a
1 changed files with 1 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
bindings/python/py_src/tokenizers/implementations/sentencepiece_bpe.py
View File

@@ -49,7 +49,7 @@ class SentencePieceBPETokenizer(BaseTokenizer):
@staticmethod
def from_file(vocab_filename: str, merges_filename: str, **kwargs):
vocab, merges = BPE.read_files(vocab_filename, merges_filename)
vocab, merges = BPE.read_file(vocab_filename, merges_filename)
return SentencePieceBPETokenizer(vocab, merges, **kwargs)
def train(