mii/tokenizers
1
0
Fork 0
mirror of https://github.com/mii443/tokenizers.git synced 2025-12-08 13:48:19 +00:00
Code Issues Packages Projects Releases Wiki Activity

Python - Update bindings

Browse Source
This commit is contained in:
Anthony MOI
2020-03-09 18:37:03 -04:00
parent 6a50ecfa5c
commit 7e9003ccb7
10 changed files with 43 additions and 67 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
bindings/python/examples/example.py
View File

@@ -60,10 +60,8 @@ if args.type == "gpt2":
# Create a Tokenizer using BPE
tok_r = Tokenizer(BPE.from_files(args.vocab, args.merges))
# Use ByteLevel Normalizer
tok_r.normalizer = normalizers.ByteLevel(add_prefix_space=False)
# Use ByteLevel PreTokenizer
tok_r.pre_tokenizer = pre_tokenizers.ByteLevel()
tok_r.pre_tokenizer = pre_tokenizers.ByteLevel(add_prefix_space=False)
# Use ByteLevel Decoder
tok_r.decoder = decoders.ByteLevel()
elif args.type == "bert":