mii/tokenizers
1
0
Fork 0
mirror of https://github.com/mii443/tokenizers.git synced 2025-08-24 00:59:19 +00:00
Code Issues Packages Projects Releases Wiki Activity

Python - Handle kwargs for bert modules

Browse Source
This commit is contained in:
Anthony MOI
2019-12-13 15:28:29 -05:00
parent 3355be89cd
commit e93cc62a71
3 changed files with 35 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
bindings/python/examples/example.py
View File

@ -61,8 +61,8 @@ elif args.type == "bert":
print("Running Bert tokenizer")
tok_p = BertTokenizer.from_pretrained('bert-base-uncased')
tok_r = Tokenizer(models.WordPiece.from_files(args.vocab))
tok_r.with_pre_tokenizer(pre_tokenizers.BasicPreTokenizer.new())
tok_r = Tokenizer(models.WordPiece.from_files(args.vocab, unk_token="[UNK]", max_input_chars_per_word=100))
tok_r.with_pre_tokenizer(pre_tokenizers.BasicPreTokenizer.new(do_lower_case=True, tokenize_chinese_chars=True, never_split=[]))
tok_r.with_decoder(decoders.WordPiece.new())
else:
raise Exception(f"Unknown type {args.type}")