mii/tokenizers
1
0
Fork 0
mirror of https://github.com/mii443/tokenizers.git synced 2025-12-08 13:48:19 +00:00
Code Issues Packages Projects Releases Wiki Activity

Python - Update docs and implementations

Browse Source
This commit is contained in:
Anthony MOI
2020-03-04 16:47:05 -05:00
parent 2393506dc7
commit f1460fadb9
3 changed files with 12 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
bindings/python/examples/example.py
View File

@@ -60,8 +60,10 @@ if args.type == "gpt2":
# Create a Tokenizer using BPE
tok_r = Tokenizer(BPE.from_files(args.vocab, args.merges))
# Use ByteLevel Normalizer
tok_r.normalizer = normalizers.ByteLevel(add_prefix_space=False)
# Use ByteLevel PreTokenizer
tok_r.pre_tokenizer = pre_tokenizers.ByteLevel(add_prefix_space=False)
tok_r.pre_tokenizer = pre_tokenizers.ByteLevel()
# Use ByteLevel Decoder
tok_r.decoder = decoders.ByteLevel()
elif args.type == "bert":