mii/tokenizers
1
0
Fork 0
mirror of https://github.com/mii443/tokenizers.git synced 2025-08-24 09:09:21 +00:00
Code Issues Packages Projects Releases Wiki Activity

Python - Update examples to use new models API

Browse Source
This commit is contained in:
Bjarte Johansen
2020-04-06 21:18:28 +02:00
parent 823066fea9
commit fab97475e5
2 changed files with 4 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
bindings/python/examples/custom_pre_tokenizer.py
View File

@ -41,8 +41,8 @@ def tokenize(sentence):
# Create a Tokenizer using a BPE model
bpe = models.BPE.from_files(args.vocab, args.merges)
tokenizer = Tokenizer(models.BPE.from_files(args.vocab, args.merges))
bpe = models.BPE(args.vocab, args.merges)
tokenizer = Tokenizer(bpe)
# Test the good custom classes
good_custom = GoodCustom()