mirror of
https://github.com/mii443/tokenizers.git
synced 2025-08-23 00:35:35 +00:00
5 lines
113 B
Python
5 lines
113 B
Python
from tokenizers import Tokenizer
|
|
|
|
|
|
tokenizer = Tokenizer.from_file("/home/nicolas/Downloads/tokenizer-wiki.txt")
|