mii/tokenizers
1
0
Fork 0
mirror of https://github.com/mii443/tokenizers.git synced 2025-08-23 16:49:27 +00:00
Code Issues Packages Projects Releases Wiki Activity

Python - fix example

Browse Source
This commit is contained in:
Anthony MOI
2019-12-24 11:20:40 -05:00
parent cf0e8917cd
commit 4bc5a7bbe7
1 changed files with 2 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
bindings/python/examples/example.py
View File

@ -54,7 +54,7 @@ if args.type == "gpt2":
# Create a Tokenizer using BPE
tok_r = Tokenizer(models.BPE.from_files(args.vocab, args.merges))
# Use ByteLevel PreTokenizer
tok_r.with_pre_tokenizer(pre_tokenizers.ByteLevel.new())
tok_r.with_pre_tokenizer(pre_tokenizers.ByteLevel.new(False))
# Use ByteLevel Decoder
tok_r.with_decoder(decoders.ByteLevel.new())
elif args.type == "bert":
@ -109,7 +109,7 @@ for i in range(0, len(encoded_r)):
print("")
print(f"Ids differences: {diff_ids}")
decoded_r = tok_r.decode_batch([ sentence.ids for sentence in encoded_r ])
decoded_r = tok_r.decode_batch([ sentence.ids for sentence in encoded_r ], False)
decoded_p = [ tok_p.decode(en) for en in encoded_p ]
diff_decoded = 0
for i in range(0, len(text)):