mii/tokenizers
1
0
Fork 0
mirror of https://github.com/mii443/tokenizers.git synced 2025-12-09 06:08:22 +00:00
Code Issues Packages Projects Releases Wiki Activity

Update python example

Browse Source
This commit is contained in:
Anthony MOI
2019-11-21 11:57:57 -05:00
parent c28a83cdc4
commit 070fd08583
1 changed files with 3 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
bindings/python/example.py
View File

@@ -69,4 +69,6 @@ print(f"Transformer tokenizer took: {end - start} sec")
ids_r = [ [ token.id for token in sentence ] for sentence in encoded_r ] ids_r = [ [ token.id for token in sentence ] for sentence in encoded_r ]
assert(ids_r == encoded_p) assert(ids_r == encoded_p)
print(f"Decoded sentences: {tok_r.decode_batch(ids_r)}") decoded_r = tok_r.decode_batch(ids_r)
print(f"Decoded sentences: {decoded_r}")
assert(decoded_r == text)