mii/tokenizers
1
0
Fork 0
mirror of https://github.com/mii443/tokenizers.git synced 2025-12-08 13:48:19 +00:00
Code Issues Packages Projects Releases Wiki Activity

Python - hotfix BertWordPieceTokenizer decoder

Browse Source
This commit is contained in:
Anthony MOI
2020-02-13 16:30:52 -05:00
parent a28d83a1b1
commit 5bd93ee822
1 changed files with 1 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
bindings/python/tokenizers/implementations/bert_wordpiece.py
View File

@@ -50,7 +50,7 @@ class BertWordPieceTokenizer(BaseTokenizer):
tokenizer.post_processor = BertProcessing( tokenizer.post_processor = BertProcessing(
(sep_token, sep_token_id), (cls_token, cls_token_id) (sep_token, sep_token_id), (cls_token, cls_token_id)
) )
tokenizer.decoders = decoders.WordPiece(prefix=wordpieces_prefix) tokenizer.decoder = decoders.WordPiece(prefix=wordpieces_prefix)
parameters = { parameters = {
"model": "BertWordPiece", "model": "BertWordPiece",