mii/tokenizers
1
0
Fork 0
mirror of https://github.com/mii443/tokenizers.git synced 2025-08-26 01:59:49 +00:00
Code Issues Packages Projects Releases Wiki Activity

Python - Update examples with getter/setter

Browse Source
This commit is contained in:
Anthony MOI
2020-01-07 15:23:11 -05:00
parent 8bbf832842
commit cc33418044
4 changed files with 19 additions and 19 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
bindings/python/examples/custom_pre_tokenizer.py
View File

@ -44,8 +44,8 @@ good_custom = GoodCustom()
good_pretok = pre_tokenizers.PreTokenizer.custom(good_custom)
good_decoder = decoders.Decoder.custom(good_custom)
tokenizer.with_pre_tokenizer(good_pretok)
tokenizer.with_decoder(good_decoder)
tokenizer.pre_tokenizer = good_pretok
tokenizer.decoder = good_decoder
print("Tokenization will work with good custom:")
encoding = tokenizer.encode("Hey friend!")
@ -60,8 +60,8 @@ bad_custom = BadCustom()
bad_pretok = pre_tokenizers.PreTokenizer.custom(bad_custom)
bad_decoder = decoders.Decoder.custom(bad_custom)
tokenizer.with_pre_tokenizer(bad_pretok)
tokenizer.with_decoder(bad_decoder)
tokenizer.pre_tokenizer = bad_pretok
tokenizer.decoder = bad_decoder
try:
encoding = tokenizer.encode("Hey friend!")
except: