mirror of
https://github.com/mii443/tokenizers.git
synced 2025-08-23 16:49:27 +00:00
4 lines
92 B
Python
4 lines
92 B
Python
from tokenizers import WhitespaceTokenizer
|
|
|
|
print(WhitespaceTokenizer.tokenize("Hey man!"))
|